From 94cb1e9b5ac364db43838fe93a026ebad90f4f8c Mon Sep 17 00:00:00 2001 From: nym21 Date: Tue, 28 Jul 2026 21:35:02 +0200 Subject: [PATCH] next: ai part 9 --- crates/brk_client/src/lib.rs | 495 +++++- .../src/cointime/activity/compute.rs | 58 - .../src/cointime/supply/compute.rs | 47 - .../src/cointime/supply/import.rs | 19 - .../brk_computer/src/cointime/supply/vecs.rs | 10 - .../src/distribution/addr/supply/share.rs | 10 +- .../src/distribution/metrics/outputs/base.rs | 6 +- .../src/frameworks/coinflow/compute.rs | 562 ++++++ .../src/frameworks/coinflow/import.rs | 94 + .../src/frameworks/coinflow/mod.rs | 79 + .../src/frameworks/coinflow/vecs.rs | 129 ++ .../frameworks/cointime/activity/compute.rs | 70 + .../cointime/activity/import.rs | 46 +- .../src/frameworks/cointime/activity/mod.rs | 6 + .../cointime/activity/vecs.rs | 15 +- .../cointime/adjusted/compute.rs | 0 .../cointime/adjusted/import.rs | 0 .../cointime/adjusted}/mod.rs | 0 .../cointime/adjusted/vecs.rs | 0 .../frameworks/cointime/age_range/compute.rs | 402 +++++ .../frameworks/cointime/age_range/import.rs | 66 + .../cointime/age_range}/mod.rs | 2 +- .../src/frameworks/cointime/age_range/vecs.rs | 25 + .../{ => frameworks}/cointime/cap/compute.rs | 0 .../{ => frameworks}/cointime/cap/import.rs | 0 .../cointime/cap}/mod.rs | 0 .../src/{ => frameworks}/cointime/cap/vecs.rs | 0 .../src/{ => frameworks}/cointime/compute.rs | 17 +- .../src/{ => frameworks}/cointime/import.rs | 6 +- .../src/{ => frameworks}/cointime/mod.rs | 5 +- .../cointime/prices/compute.rs | 0 .../cointime/prices/import.rs | 0 .../cap => frameworks/cointime/prices}/mod.rs | 0 .../{ => frameworks}/cointime/prices/vecs.rs | 0 .../cointime/reserve_risk/compute.rs | 0 .../cointime/reserve_risk/import.rs | 0 .../cointime/reserve_risk}/mod.rs | 0 .../cointime/reserve_risk/vecs.rs | 0 .../src/frameworks/cointime/supply/compute.rs | 168 ++ .../src/frameworks/cointime/supply/import.rs | 49 + .../{ => frameworks}/cointime/supply/mod.rs | 2 +- .../src/frameworks/cointime/supply/vecs.rs | 22 + .../cointime/value/compute.rs | 0 .../{ => frameworks}/cointime/value/import.rs | 0 .../cointime/value}/mod.rs | 0 .../{ => frameworks}/cointime/value/vecs.rs | 0 crates/brk_computer/src/frameworks/mod.rs | 29 + crates/brk_computer/src/internal/db_utils.rs | 17 +- .../per_block/percent/cumulative_rolling.rs | 10 +- .../internal/per_block/ratio/percentiles.rs | 6 +- .../src/internal/transform/mod.rs | 8 +- crates/brk_computer/src/lib.rs | 60 +- .../src/mining/rewards/compute.rs | 4 +- .../src/models/bedrock/compute.rs | 646 +++++++ .../brk_computer/src/models/bedrock/import.rs | 105 ++ crates/brk_computer/src/models/bedrock/mod.rs | 8 + .../src/models/bedrock/urpd_metric.rs | 570 ++++++ .../brk_computer/src/models/bedrock/vecs.rs | 95 + crates/brk_computer/src/models/mod.rs | 1 + crates/brk_computer/src/op_return/compute.rs | 5 +- crates/brk_computer/src/pools/major.rs | 4 +- crates/brk_computer/src/supply/import.rs | 4 +- .../src/stores/addr_tracker/mod.rs | 11 +- crates/brk_query/src/impl/addr/stats.rs | 1 + crates/brk_types/src/addr_chain_stats.rs | 3 + crates/brk_types/src/addr_mempool_stats.rs | 9 +- crates/brk_types/src/sats_signed.rs | 1 + modules/brk-client/index.js | 485 +++++- packages/brk_client/brk_client/__init__.py | 252 ++- website/llms-full.txt | 8 +- website/llms.txt | 2 +- website_next/ask/compactor.js | 5 +- website_next/ask/context.js | 12 +- website_next/ask/index.js | 7 +- website_next/ask/storage.js | 16 +- website_next/ask/tools/api/answer.js | 753 ++++---- website_next/ask/tools/api/execute.js | 2 +- website_next/ask/tools/api/index.js | 3 + website_next/ask/tools/api/openapi.js | 1 + website_next/ask/tools/api/routing.js | 315 ++-- website_next/ask/tools/api/worker.js | 18 +- website_next/ask/tools/chart.js | 11 +- website_next/ask/tools/data.js | 30 +- website_next/ask/tools/direct/chart.js | 66 - website_next/ask/tools/direct/evidence.js | 19 - website_next/ask/tools/direct/request.js | 96 - website_next/ask/tools/history.js | 79 - website_next/ask/tools/index.js | 836 +++++---- website_next/ask/tools/metrics/index.js | 13 +- website_next/ask/tools/metrics/language.js | 41 - website_next/ask/tools/metrics/options.js | 109 -- website_next/ask/tools/metrics/unit.js | 11 + website_next/ask/tools/metrics/worker.js | 233 ++- website_next/ask/tools/prompts.js | 57 - website_next/ask/tools/refs.js | 16 +- website_next/ask/tools/render.js | 252 +-- website_next/ask/tools/schemas.js | 170 -- website_next/ask/tools/session.js | 1542 ----------------- .../ask/tools/session/capabilities.js | 540 ++++++ website_next/ask/tools/session/context.js | 69 + website_next/ask/tools/session/evidence.js | 307 ++++ website_next/ask/tools/session/executor.js | 546 ++++++ website_next/ask/tools/session/index.js | 570 ++++++ website_next/ask/tools/source/answer.js | 142 -- website_next/ask/tools/source/formula.js | 339 ---- website_next/ask/tools/source/index.js | 26 +- website_next/ask/tools/source/search.js | 300 ++-- website_next/ask/tools/source/worker.js | 76 +- website_next/ask/tools/text.js | 5 +- website_next/ask/worker.js | 105 +- website_next/llms-full.txt | 8 +- website_next/llms.txt | 2 +- 112 files changed, 8064 insertions(+), 4438 deletions(-) delete mode 100644 crates/brk_computer/src/cointime/activity/compute.rs delete mode 100644 crates/brk_computer/src/cointime/supply/compute.rs delete mode 100644 crates/brk_computer/src/cointime/supply/import.rs delete mode 100644 crates/brk_computer/src/cointime/supply/vecs.rs create mode 100644 crates/brk_computer/src/frameworks/coinflow/compute.rs create mode 100644 crates/brk_computer/src/frameworks/coinflow/import.rs create mode 100644 crates/brk_computer/src/frameworks/coinflow/mod.rs create mode 100644 crates/brk_computer/src/frameworks/coinflow/vecs.rs create mode 100644 crates/brk_computer/src/frameworks/cointime/activity/compute.rs rename crates/brk_computer/src/{ => frameworks}/cointime/activity/import.rs (58%) create mode 100644 crates/brk_computer/src/frameworks/cointime/activity/mod.rs rename crates/brk_computer/src/{ => frameworks}/cointime/activity/vecs.rs (70%) rename crates/brk_computer/src/{ => frameworks}/cointime/adjusted/compute.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/adjusted/import.rs (100%) rename crates/brk_computer/src/{cointime/activity => frameworks/cointime/adjusted}/mod.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/adjusted/vecs.rs (100%) create mode 100644 crates/brk_computer/src/frameworks/cointime/age_range/compute.rs create mode 100644 crates/brk_computer/src/frameworks/cointime/age_range/import.rs rename crates/brk_computer/src/{cointime/value => frameworks/cointime/age_range}/mod.rs (51%) create mode 100644 crates/brk_computer/src/frameworks/cointime/age_range/vecs.rs rename crates/brk_computer/src/{ => frameworks}/cointime/cap/compute.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/cap/import.rs (100%) rename crates/brk_computer/src/{cointime/adjusted => frameworks/cointime/cap}/mod.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/cap/vecs.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/compute.rs (80%) rename crates/brk_computer/src/{ => frameworks}/cointime/import.rs (85%) rename crates/brk_computer/src/{ => frameworks}/cointime/mod.rs (83%) rename crates/brk_computer/src/{ => frameworks}/cointime/prices/compute.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/prices/import.rs (100%) rename crates/brk_computer/src/{cointime/cap => frameworks/cointime/prices}/mod.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/prices/vecs.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/reserve_risk/compute.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/reserve_risk/import.rs (100%) rename crates/brk_computer/src/{cointime/prices => frameworks/cointime/reserve_risk}/mod.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/reserve_risk/vecs.rs (100%) create mode 100644 crates/brk_computer/src/frameworks/cointime/supply/compute.rs create mode 100644 crates/brk_computer/src/frameworks/cointime/supply/import.rs rename crates/brk_computer/src/{ => frameworks}/cointime/supply/mod.rs (52%) create mode 100644 crates/brk_computer/src/frameworks/cointime/supply/vecs.rs rename crates/brk_computer/src/{ => frameworks}/cointime/value/compute.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/value/import.rs (100%) rename crates/brk_computer/src/{cointime/reserve_risk => frameworks/cointime/value}/mod.rs (100%) rename crates/brk_computer/src/{ => frameworks}/cointime/value/vecs.rs (100%) create mode 100644 crates/brk_computer/src/frameworks/mod.rs create mode 100644 crates/brk_computer/src/models/bedrock/compute.rs create mode 100644 crates/brk_computer/src/models/bedrock/import.rs create mode 100644 crates/brk_computer/src/models/bedrock/mod.rs create mode 100644 crates/brk_computer/src/models/bedrock/urpd_metric.rs create mode 100644 crates/brk_computer/src/models/bedrock/vecs.rs create mode 100644 crates/brk_computer/src/models/mod.rs delete mode 100644 website_next/ask/tools/direct/chart.js delete mode 100644 website_next/ask/tools/direct/evidence.js delete mode 100644 website_next/ask/tools/direct/request.js delete mode 100644 website_next/ask/tools/history.js delete mode 100644 website_next/ask/tools/metrics/language.js delete mode 100644 website_next/ask/tools/metrics/options.js create mode 100644 website_next/ask/tools/metrics/unit.js delete mode 100644 website_next/ask/tools/prompts.js delete mode 100644 website_next/ask/tools/schemas.js delete mode 100644 website_next/ask/tools/session.js create mode 100644 website_next/ask/tools/session/capabilities.js create mode 100644 website_next/ask/tools/session/context.js create mode 100644 website_next/ask/tools/session/evidence.js create mode 100644 website_next/ask/tools/session/executor.js create mode 100644 website_next/ask/tools/session/index.js delete mode 100644 website_next/ask/tools/source/answer.js delete mode 100644 website_next/ask/tools/source/formula.js diff --git a/crates/brk_client/src/lib.rs b/crates/brk_client/src/lib.rs index e28c00ce5..107d02db8 100644 --- a/crates/brk_client/src/lib.rs +++ b/crates/brk_client/src/lib.rs @@ -1663,6 +1663,36 @@ impl CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2 { } } +/// Pattern struct for repeated tree structure. +pub struct Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern { + pub pct10: SeriesPattern1, + pub pct20: SeriesPattern1, + pub pct30: SeriesPattern1, + pub pct40: SeriesPattern1, + pub pct50: SeriesPattern1, + pub pct60: SeriesPattern1, + pub pct70: SeriesPattern1, + pub pct80: SeriesPattern1, + pub pct90: SeriesPattern1, +} + +impl Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + pct10: SeriesPattern1::new(client.clone(), _m(&acc, "pct10")), + pct20: SeriesPattern1::new(client.clone(), _m(&acc, "pct20")), + pct30: SeriesPattern1::new(client.clone(), _m(&acc, "pct30")), + pct40: SeriesPattern1::new(client.clone(), _m(&acc, "pct40")), + pct50: SeriesPattern1::new(client.clone(), _m(&acc, "pct50")), + pct60: SeriesPattern1::new(client.clone(), _m(&acc, "pct60")), + pct70: SeriesPattern1::new(client.clone(), _m(&acc, "pct70")), + pct80: SeriesPattern1::new(client.clone(), _m(&acc, "pct80")), + pct90: SeriesPattern1::new(client.clone(), _m(&acc, "pct90")), + } + } +} + /// Pattern struct for repeated tree structure. pub struct CentsPercentilesRatioRawSatsSmaStdUsdPattern { pub cents: SeriesPattern1, @@ -1771,7 +1801,7 @@ pub struct ActivityCostInvestedOutputsRealizedSupplyUnrealizedPattern2 { pub activity: CoindaysCoinyearsDormancyTransferPattern, pub cost_basis: InMaxMinPerSupplyPattern, pub invested_capital: InPattern, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub realized: CapCapitalizedGrossLossMvrvNetPeakPriceProfitSellSoprPattern2, pub supply: DeltaDominanceHalfInTotalPattern2, pub unrealized: CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2, @@ -1803,6 +1833,32 @@ impl CapLossMvrvNetPriceProfitSoprPattern { } } +/// Pattern struct for repeated tree structure. +pub struct CoindaysLivelinessRatioSupplyVaultednessPattern { + pub coindays_consumed: AverageBlockCumulativeSumPattern, + pub coindays_created: AverageBlockCumulativeSumPattern, + pub coindays_stored: AverageBlockCumulativeSumPattern, + pub liveliness: SeriesPattern1, + pub ratio: SeriesPattern1, + pub supply: ActiveVaultedPattern, + pub vaultedness: SeriesPattern1, +} + +impl CoindaysLivelinessRatioSupplyVaultednessPattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + coindays_consumed: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_consumed")), + coindays_created: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_created")), + coindays_stored: AverageBlockCumulativeSumPattern::new(client.clone(), _m(&acc, "coindays_stored")), + liveliness: SeriesPattern1::new(client.clone(), _m(&acc, "liveliness")), + ratio: SeriesPattern1::new(client.clone(), _m(&acc, "activity_to_vaultedness")), + supply: ActiveVaultedPattern::new(client.clone(), acc.clone()), + vaultedness: SeriesPattern1::new(client.clone(), _m(&acc, "vaultedness")), + } + } +} + /// Pattern struct for repeated tree structure. pub struct InMaxMinPerSupplyPattern { pub in_loss: PerPattern, @@ -1909,7 +1965,7 @@ impl _1m1w1y2y4yAllPattern { pub struct ActivityAddrOutputsRealizedSupplyUnrealizedPattern { pub activity: TransferPattern, pub addr_count: BaseDeltaPattern, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub realized: CapLossMvrvPriceProfitPattern, pub supply: DeltaDominanceTotalPattern, pub unrealized: NuplPattern, @@ -1921,7 +1977,7 @@ impl ActivityAddrOutputsRealizedSupplyUnrealizedPattern { Self { activity: TransferPattern::new(client.clone(), _m(&acc, "transfer_volume")), addr_count: BaseDeltaPattern::new(client.clone(), _m(&acc, "addr_count")), - outputs: SpendingSpentUnspentPattern::new(client.clone(), acc.clone()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), acc.clone()), realized: CapLossMvrvPriceProfitPattern::new(client.clone(), acc.clone()), supply: DeltaDominanceTotalPattern::new(client.clone(), _m(&acc, "supply")), unrealized: NuplPattern::new(client.clone(), _m(&acc, "nupl")), @@ -2142,7 +2198,7 @@ impl ActiveBidirectionalReactivatedReceivingSendingPattern { /// Pattern struct for repeated tree structure. pub struct ActivityOutputsRealizedSupplyUnrealizedPattern { pub activity: CoindaysTransferPattern, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub realized: CapLossMvrvNetPriceProfitSoprPattern, pub supply: DeltaDominanceHalfInTotalPattern, pub unrealized: LossNetNuplProfitPattern, @@ -2153,7 +2209,7 @@ impl ActivityOutputsRealizedSupplyUnrealizedPattern { pub fn new(client: Arc, acc: String) -> Self { Self { activity: CoindaysTransferPattern::new(client.clone(), acc.clone()), - outputs: SpendingSpentUnspentPattern::new(client.clone(), acc.clone()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), acc.clone()), realized: CapLossMvrvNetPriceProfitSoprPattern::new(client.clone(), acc.clone()), supply: DeltaDominanceHalfInTotalPattern::new(client.clone(), _m(&acc, "supply")), unrealized: LossNetNuplProfitPattern::new(client.clone(), acc.clone()), @@ -2164,7 +2220,7 @@ impl ActivityOutputsRealizedSupplyUnrealizedPattern { /// Pattern struct for repeated tree structure. pub struct ActivityOutputsRealizedSupplyUnrealizedPattern3 { pub activity: TransferPattern, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub realized: CapLossMvrvPriceProfitPattern, pub supply: DeltaDominanceHalfInTotalPattern, pub unrealized: LossNuplProfitPattern, @@ -2175,7 +2231,7 @@ impl ActivityOutputsRealizedSupplyUnrealizedPattern3 { pub fn new(client: Arc, acc: String) -> Self { Self { activity: TransferPattern::new(client.clone(), _m(&acc, "transfer_volume")), - outputs: SpendingSpentUnspentPattern::new(client.clone(), acc.clone()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), acc.clone()), realized: CapLossMvrvPriceProfitPattern::new(client.clone(), acc.clone()), supply: DeltaDominanceHalfInTotalPattern::new(client.clone(), _m(&acc, "supply")), unrealized: LossNuplProfitPattern::new(client.clone(), acc.clone()), @@ -2186,7 +2242,7 @@ impl ActivityOutputsRealizedSupplyUnrealizedPattern3 { /// Pattern struct for repeated tree structure. pub struct ActivityOutputsRealizedSupplyUnrealizedPattern2 { pub activity: TransferPattern, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub realized: CapLossMvrvPriceProfitPattern, pub supply: DeltaDominanceTotalPattern, pub unrealized: NuplPattern, @@ -2197,7 +2253,7 @@ impl ActivityOutputsRealizedSupplyUnrealizedPattern2 { pub fn new(client: Arc, acc: String) -> Self { Self { activity: TransferPattern::new(client.clone(), _m(&acc, "transfer_volume")), - outputs: SpendingSpentUnspentPattern::new(client.clone(), acc.clone()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), acc.clone()), realized: CapLossMvrvPriceProfitPattern::new(client.clone(), acc.clone()), supply: DeltaDominanceTotalPattern::new(client.clone(), _m(&acc, "supply")), unrealized: NuplPattern::new(client.clone(), _m(&acc, "nupl")), @@ -2249,6 +2305,15 @@ impl BtcCentsDeltaSatsUsdPattern { } } +/// Pattern struct for repeated tree structure. +pub struct BtcCentsInSatsUsdPattern { + pub btc: SeriesPattern1, + pub cents: SeriesPattern1, + pub in_loss: SharePattern2, + pub sats: SeriesPattern1, + pub usd: SeriesPattern1, +} + /// Pattern struct for repeated tree structure. pub struct BtcCentsSatsShareUsdPattern { pub btc: SeriesPattern1, @@ -2368,6 +2433,28 @@ impl PhsReboundThsPattern { } } +/// Pattern struct for repeated tree structure. +pub struct Pct95Pct98Pct99Pattern { + pub pct95: SeriesPattern1, + pub pct98: SeriesPattern1, + pub pct99: SeriesPattern1, + pub pct99_5: SeriesPattern1, + pub pct99_9: SeriesPattern1, +} + +impl Pct95Pct98Pct99Pattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + pct95: SeriesPattern1::new(client.clone(), _m(&acc, "pct95")), + pct98: SeriesPattern1::new(client.clone(), _m(&acc, "pct98")), + pct99: SeriesPattern1::new(client.clone(), _m(&acc, "pct99")), + pct99_5: SeriesPattern1::new(client.clone(), _m(&acc, "pct99_5")), + pct99_9: SeriesPattern1::new(client.clone(), _m(&acc, "pct99_9")), + } + } +} + /// Pattern struct for repeated tree structure. pub struct _1m1w1y24hPattern4 { pub _1m: BtcCentsSatsUsdPattern, @@ -2756,6 +2843,26 @@ impl LossNetNuplProfitPattern { } } +/// Pattern struct for repeated tree structure. +pub struct MobilitySpendingSupplyPattern { + pub mobility: SeriesPattern1, + pub spending_exposure: SeriesPattern1, + pub spending_rate: SeriesPattern1, + pub supply: ImmobileMobilePattern, +} + +impl MobilitySpendingSupplyPattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + mobility: SeriesPattern1::new(client.clone(), _m(&acc, "mobility")), + spending_exposure: SeriesPattern1::new(client.clone(), _m(&acc, "spending_exposure")), + spending_rate: SeriesPattern1::new(client.clone(), _m(&acc, "spending_rate")), + supply: ImmobileMobilePattern::new(client.clone(), acc.clone()), + } + } +} + /// Pattern struct for repeated tree structure. pub struct NuplRealizedSupplyUnrealizedPattern { pub nupl: RatioRawPattern, @@ -2985,6 +3092,24 @@ impl DeltaDominanceTotalPattern { } } +/// Pattern struct for repeated tree structure. +pub struct FloorLevelLossPattern { + pub floor: Pct95Pct98Pct99Pattern, + pub level: Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern, + pub loss_threshold: Pct95Pct98Pct99Pattern, +} + +impl FloorLevelLossPattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + floor: Pct95Pct98Pct99Pattern::new(client.clone(), _m(&acc, "floor")), + level: Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern::new(client.clone(), _m(&acc, "level")), + loss_threshold: Pct95Pct98Pct99Pattern::new(client.clone(), _m(&acc, "loss_threshold")), + } + } +} + /// Pattern struct for repeated tree structure. pub struct GreedNetPainPattern { pub greed_index: CentsUsdPattern3, @@ -3130,19 +3255,19 @@ impl RsiStochPattern { } /// Pattern struct for repeated tree structure. -pub struct SpendingSpentUnspentPattern { - pub spending_rate: SeriesPattern1, +pub struct SpentUnspentUtxoPattern { pub spent_count: AverageBlockCumulativeSumPattern2, pub unspent_count: BaseDeltaPattern, + pub utxo_turnover_1y: SeriesPattern1, } -impl SpendingSpentUnspentPattern { +impl SpentUnspentUtxoPattern { /// Create a new pattern node with accumulated series name. pub fn new(client: Arc, acc: String) -> Self { Self { - spending_rate: SeriesPattern1::new(client.clone(), _m(&acc, "spending_rate")), spent_count: AverageBlockCumulativeSumPattern2::new(client.clone(), _m(&acc, "spent_utxo_count")), unspent_count: BaseDeltaPattern::new(client.clone(), _m(&acc, "utxo_count")), + utxo_turnover_1y: SeriesPattern1::new(client.clone(), _m(&acc, "utxo_turnover_1y")), } } } @@ -3213,6 +3338,22 @@ impl AbsoluteRatePattern3 { } } +/// Pattern struct for repeated tree structure. +pub struct ActiveVaultedPattern { + pub active: BtcCentsSatsUsdPattern, + pub vaulted: BtcCentsSatsUsdPattern, +} + +impl ActiveVaultedPattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + active: BtcCentsSatsUsdPattern::new(client.clone(), _m(&acc, "active_supply")), + vaulted: BtcCentsSatsUsdPattern::new(client.clone(), _m(&acc, "vaulted_supply")), + } + } +} + /// Pattern struct for repeated tree structure. pub struct AddrUtxoPattern { pub addr: BtcCentsSatsUsdPattern, @@ -3437,6 +3578,22 @@ impl FundedTotalPattern { } } +/// Pattern struct for repeated tree structure. +pub struct ImmobileMobilePattern { + pub immobile: BtcCentsSatsUsdPattern, + pub mobile: BtcCentsSatsUsdPattern, +} + +impl ImmobileMobilePattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + immobile: BtcCentsSatsUsdPattern::new(client.clone(), _m(&acc, "immobile_supply")), + mobile: BtcCentsSatsUsdPattern::new(client.clone(), _m(&acc, "mobile_supply")), + } + } +} + /// Pattern struct for repeated tree structure. pub struct InPattern2 { pub in_loss: CentsUsdPattern3, @@ -3617,6 +3774,20 @@ impl _24hPattern { } } +/// Pattern struct for repeated tree structure. +pub struct InPattern3 { + pub in_loss: SharePattern2, +} + +impl InPattern3 { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + in_loss: SharePattern2::new(client.clone(), acc.clone()), + } + } +} + /// Pattern struct for repeated tree structure. pub struct NuplPattern { pub nupl: RatioRawPattern, @@ -3659,6 +3830,34 @@ impl SharePattern { } } +/// Pattern struct for repeated tree structure. +pub struct SharePattern2 { + pub share: SeriesPattern1, +} + +impl SharePattern2 { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + share: SeriesPattern1::new(client.clone(), acc.clone()), + } + } +} + +/// Pattern struct for repeated tree structure. +pub struct SupplyPattern { + pub supply: InPattern3, +} + +impl SupplyPattern { + /// Create a new pattern node with accumulated series name. + pub fn new(client: Arc, acc: String) -> Self { + Self { + supply: InPattern3::new(client.clone(), acc.clone()), + } + } +} + /// Pattern struct for repeated tree structure. pub struct TransferPattern { pub transfer_volume: AverageBlockCumulativeSumPattern3, @@ -3686,6 +3885,8 @@ pub struct SeriesTree { pub op_return: SeriesTree_OpReturn, pub mining: SeriesTree_Mining, pub cointime: SeriesTree_Cointime, + pub coinflow: SeriesTree_Coinflow, + pub bedrock: SeriesTree_Bedrock, pub constants: SeriesTree_Constants, pub indexes: SeriesTree_Indexes, pub indicators: SeriesTree_Indicators, @@ -3709,6 +3910,8 @@ impl SeriesTree { op_return: SeriesTree_OpReturn::new(client.clone(), format!("{base_path}_op_return")), mining: SeriesTree_Mining::new(client.clone(), format!("{base_path}_mining")), cointime: SeriesTree_Cointime::new(client.clone(), format!("{base_path}_cointime")), + coinflow: SeriesTree_Coinflow::new(client.clone(), format!("{base_path}_coinflow")), + bedrock: SeriesTree_Bedrock::new(client.clone(), format!("{base_path}_bedrock")), constants: SeriesTree_Constants::new(client.clone(), format!("{base_path}_constants")), indexes: SeriesTree_Indexes::new(client.clone(), format!("{base_path}_indexes")), indicators: SeriesTree_Indicators::new(client.clone(), format!("{base_path}_indicators")), @@ -5651,6 +5854,7 @@ impl SeriesTree_Mining_Hashrate_Rate_Sma { /// Series tree node. pub struct SeriesTree_Cointime { pub activity: SeriesTree_Cointime_Activity, + pub age_range: SeriesTree_Cointime_AgeRange, pub supply: SeriesTree_Cointime_Supply, pub value: SeriesTree_Cointime_Value, pub cap: SeriesTree_Cointime_Cap, @@ -5663,6 +5867,7 @@ impl SeriesTree_Cointime { pub fn new(client: Arc, base_path: String) -> Self { Self { activity: SeriesTree_Cointime_Activity::new(client.clone(), format!("{base_path}_activity")), + age_range: SeriesTree_Cointime_AgeRange::new(client.clone(), format!("{base_path}_age_range")), supply: SeriesTree_Cointime_Supply::new(client.clone(), format!("{base_path}_supply")), value: SeriesTree_Cointime_Value::new(client.clone(), format!("{base_path}_value")), cap: SeriesTree_Cointime_Cap::new(client.clone(), format!("{base_path}_cap")), @@ -5696,17 +5901,91 @@ impl SeriesTree_Cointime_Activity { } } +/// Series tree node. +pub struct SeriesTree_Cointime_AgeRange { + pub under_1h: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _1h_to_1d: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _1d_to_1w: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _1w_to_1m: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _1m_to_2m: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _2m_to_3m: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _3m_to_4m: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _4m_to_5m: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _5m_to_6m: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _6m_to_1y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _1y_to_2y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _2y_to_3y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _3y_to_4y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _4y_to_5y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _5y_to_6y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _6y_to_7y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _7y_to_8y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _8y_to_10y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _10y_to_12y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub _12y_to_15y: CoindaysLivelinessRatioSupplyVaultednessPattern, + pub over_15y: CoindaysLivelinessRatioSupplyVaultednessPattern, +} + +impl SeriesTree_Cointime_AgeRange { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + under_1h: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_under_1h_old".to_string()), + _1h_to_1d: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1h_to_1d_old".to_string()), + _1d_to_1w: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1d_to_1w_old".to_string()), + _1w_to_1m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1w_to_1m_old".to_string()), + _1m_to_2m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1m_to_2m_old".to_string()), + _2m_to_3m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_2m_to_3m_old".to_string()), + _3m_to_4m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_3m_to_4m_old".to_string()), + _4m_to_5m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_4m_to_5m_old".to_string()), + _5m_to_6m: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_5m_to_6m_old".to_string()), + _6m_to_1y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_6m_to_1y_old".to_string()), + _1y_to_2y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_1y_to_2y_old".to_string()), + _2y_to_3y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_2y_to_3y_old".to_string()), + _3y_to_4y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_3y_to_4y_old".to_string()), + _4y_to_5y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_4y_to_5y_old".to_string()), + _5y_to_6y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_5y_to_6y_old".to_string()), + _6y_to_7y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_6y_to_7y_old".to_string()), + _7y_to_8y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_7y_to_8y_old".to_string()), + _8y_to_10y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_8y_to_10y_old".to_string()), + _10y_to_12y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_10y_to_12y_old".to_string()), + _12y_to_15y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_12y_to_15y_old".to_string()), + over_15y: CoindaysLivelinessRatioSupplyVaultednessPattern::new(client.clone(), "utxos_over_15y_old".to_string()), + } + } +} + /// Series tree node. pub struct SeriesTree_Cointime_Supply { pub vaulted: BtcCentsSatsUsdPattern, - pub active: BtcCentsSatsUsdPattern, + pub active: SeriesTree_Cointime_Supply_Active, } impl SeriesTree_Cointime_Supply { pub fn new(client: Arc, base_path: String) -> Self { Self { vaulted: BtcCentsSatsUsdPattern::new(client.clone(), "vaulted_supply".to_string()), - active: BtcCentsSatsUsdPattern::new(client.clone(), "active_supply".to_string()), + active: SeriesTree_Cointime_Supply_Active::new(client.clone(), format!("{base_path}_active")), + } + } +} + +/// Series tree node. +pub struct SeriesTree_Cointime_Supply_Active { + pub btc: SeriesPattern1, + pub sats: SeriesPattern1, + pub usd: SeriesPattern1, + pub cents: SeriesPattern1, + pub in_loss: SharePattern2, +} + +impl SeriesTree_Cointime_Supply_Active { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + btc: SeriesPattern1::new(client.clone(), "active_supply".to_string()), + sats: SeriesPattern1::new(client.clone(), "active_supply_sats".to_string()), + usd: SeriesPattern1::new(client.clone(), "active_supply_usd".to_string()), + cents: SeriesPattern1::new(client.clone(), "active_supply_cents".to_string()), + in_loss: SharePattern2::new(client.clone(), "cointime_supply_in_loss_share".to_string()), } } } @@ -5806,6 +6085,172 @@ impl SeriesTree_Cointime_ReserveRisk { } } +/// Series tree node. +pub struct SeriesTree_Coinflow { + pub age_range: SeriesTree_Coinflow_AgeRange, + pub supply: SeriesTree_Coinflow_Supply, + pub horizon: SeriesTree_Coinflow_Horizon, + pub cap: CentsUsdPattern3, + pub price: CentsRatioRawSatsUsdPattern, +} + +impl SeriesTree_Coinflow { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + age_range: SeriesTree_Coinflow_AgeRange::new(client.clone(), format!("{base_path}_age_range")), + supply: SeriesTree_Coinflow_Supply::new(client.clone(), format!("{base_path}_supply")), + horizon: SeriesTree_Coinflow_Horizon::new(client.clone(), format!("{base_path}_horizon")), + cap: CentsUsdPattern3::new(client.clone(), "coinflow_cap".to_string()), + price: CentsRatioRawSatsUsdPattern::new(client.clone(), "coinflow_price".to_string()), + } + } +} + +/// Series tree node. +pub struct SeriesTree_Coinflow_AgeRange { + pub under_1h: MobilitySpendingSupplyPattern, + pub _1h_to_1d: MobilitySpendingSupplyPattern, + pub _1d_to_1w: MobilitySpendingSupplyPattern, + pub _1w_to_1m: MobilitySpendingSupplyPattern, + pub _1m_to_2m: MobilitySpendingSupplyPattern, + pub _2m_to_3m: MobilitySpendingSupplyPattern, + pub _3m_to_4m: MobilitySpendingSupplyPattern, + pub _4m_to_5m: MobilitySpendingSupplyPattern, + pub _5m_to_6m: MobilitySpendingSupplyPattern, + pub _6m_to_1y: MobilitySpendingSupplyPattern, + pub _1y_to_2y: MobilitySpendingSupplyPattern, + pub _2y_to_3y: MobilitySpendingSupplyPattern, + pub _3y_to_4y: MobilitySpendingSupplyPattern, + pub _4y_to_5y: MobilitySpendingSupplyPattern, + pub _5y_to_6y: MobilitySpendingSupplyPattern, + pub _6y_to_7y: MobilitySpendingSupplyPattern, + pub _7y_to_8y: MobilitySpendingSupplyPattern, + pub _8y_to_10y: MobilitySpendingSupplyPattern, + pub _10y_to_12y: MobilitySpendingSupplyPattern, + pub _12y_to_15y: MobilitySpendingSupplyPattern, + pub over_15y: MobilitySpendingSupplyPattern, +} + +impl SeriesTree_Coinflow_AgeRange { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + under_1h: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_under_1h_old".to_string()), + _1h_to_1d: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1h_to_1d_old".to_string()), + _1d_to_1w: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1d_to_1w_old".to_string()), + _1w_to_1m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1w_to_1m_old".to_string()), + _1m_to_2m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1m_to_2m_old".to_string()), + _2m_to_3m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_2m_to_3m_old".to_string()), + _3m_to_4m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_3m_to_4m_old".to_string()), + _4m_to_5m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_4m_to_5m_old".to_string()), + _5m_to_6m: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_5m_to_6m_old".to_string()), + _6m_to_1y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_6m_to_1y_old".to_string()), + _1y_to_2y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_1y_to_2y_old".to_string()), + _2y_to_3y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_2y_to_3y_old".to_string()), + _3y_to_4y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_3y_to_4y_old".to_string()), + _4y_to_5y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_4y_to_5y_old".to_string()), + _5y_to_6y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_5y_to_6y_old".to_string()), + _6y_to_7y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_6y_to_7y_old".to_string()), + _7y_to_8y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_7y_to_8y_old".to_string()), + _8y_to_10y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_8y_to_10y_old".to_string()), + _10y_to_12y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_10y_to_12y_old".to_string()), + _12y_to_15y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_12y_to_15y_old".to_string()), + over_15y: MobilitySpendingSupplyPattern::new(client.clone(), "utxos_over_15y_old".to_string()), + } + } +} + +/// Series tree node. +pub struct SeriesTree_Coinflow_Supply { + pub mobile: SeriesTree_Coinflow_Supply_Mobile, + pub immobile: BtcCentsSatsUsdPattern, +} + +impl SeriesTree_Coinflow_Supply { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + mobile: SeriesTree_Coinflow_Supply_Mobile::new(client.clone(), format!("{base_path}_mobile")), + immobile: BtcCentsSatsUsdPattern::new(client.clone(), "immobile_supply".to_string()), + } + } +} + +/// Series tree node. +pub struct SeriesTree_Coinflow_Supply_Mobile { + pub btc: SeriesPattern1, + pub sats: SeriesPattern1, + pub usd: SeriesPattern1, + pub cents: SeriesPattern1, + pub in_loss: SharePattern2, +} + +impl SeriesTree_Coinflow_Supply_Mobile { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + btc: SeriesPattern1::new(client.clone(), "mobile_supply".to_string()), + sats: SeriesPattern1::new(client.clone(), "mobile_supply_sats".to_string()), + usd: SeriesPattern1::new(client.clone(), "mobile_supply_usd".to_string()), + cents: SeriesPattern1::new(client.clone(), "mobile_supply_cents".to_string()), + in_loss: SharePattern2::new(client.clone(), "coinflow_supply_in_loss_share".to_string()), + } + } +} + +/// Series tree node. +pub struct SeriesTree_Coinflow_Horizon { + pub _8y: SupplyPattern, + pub _4y: SupplyPattern, + pub _2y: SupplyPattern, + pub _1y: SupplyPattern, + pub _6m: SupplyPattern, + pub _3m: SupplyPattern, + pub _1m: SupplyPattern, +} + +impl SeriesTree_Coinflow_Horizon { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + _8y: SupplyPattern::new(client.clone(), "coinflow_8y_supply_in_loss_share".to_string()), + _4y: SupplyPattern::new(client.clone(), "coinflow_4y_supply_in_loss_share".to_string()), + _2y: SupplyPattern::new(client.clone(), "coinflow_2y_supply_in_loss_share".to_string()), + _1y: SupplyPattern::new(client.clone(), "coinflow_1y_supply_in_loss_share".to_string()), + _6m: SupplyPattern::new(client.clone(), "coinflow_6m_supply_in_loss_share".to_string()), + _3m: SupplyPattern::new(client.clone(), "coinflow_3m_supply_in_loss_share".to_string()), + _1m: SupplyPattern::new(client.clone(), "coinflow_1m_supply_in_loss_share".to_string()), + } + } +} + +/// Series tree node. +pub struct SeriesTree_Bedrock { + pub raw: FloorLevelLossPattern, + pub cointime: FloorLevelLossPattern, + pub coinflow: FloorLevelLossPattern, + pub coinflow_8y: FloorLevelLossPattern, + pub coinflow_4y: FloorLevelLossPattern, + pub coinflow_2y: FloorLevelLossPattern, + pub coinflow_1y: FloorLevelLossPattern, + pub coinflow_6m: FloorLevelLossPattern, + pub coinflow_3m: FloorLevelLossPattern, + pub coinflow_1m: FloorLevelLossPattern, +} + +impl SeriesTree_Bedrock { + pub fn new(client: Arc, base_path: String) -> Self { + Self { + raw: FloorLevelLossPattern::new(client.clone(), "bedrock_raw".to_string()), + cointime: FloorLevelLossPattern::new(client.clone(), "bedrock_cointime".to_string()), + coinflow: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow".to_string()), + coinflow_8y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_8y".to_string()), + coinflow_4y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_4y".to_string()), + coinflow_2y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_2y".to_string()), + coinflow_1y: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_1y".to_string()), + coinflow_6m: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_6m".to_string()), + coinflow_3m: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_3m".to_string()), + coinflow_1m: FloorLevelLossPattern::new(client.clone(), "bedrock_coinflow_1m".to_string()), + } + } +} + /// Series tree node. pub struct SeriesTree_Constants { pub _0: SeriesPattern1, @@ -7755,7 +8200,7 @@ impl SeriesTree_Cohorts_Utxo_All { pub struct SeriesTree_Cohorts_Utxo_All_Outputs { pub unspent_count: BaseDeltaPattern, pub spent_count: AverageBlockCumulativeSumPattern2, - pub spending_rate: SeriesPattern1, + pub utxo_turnover_1y: SeriesPattern1, } impl SeriesTree_Cohorts_Utxo_All_Outputs { @@ -7763,7 +8208,7 @@ impl SeriesTree_Cohorts_Utxo_All_Outputs { Self { unspent_count: BaseDeltaPattern::new(client.clone(), "utxo_count".to_string()), spent_count: AverageBlockCumulativeSumPattern2::new(client.clone(), "spent_utxo_count".to_string()), - spending_rate: SeriesPattern1::new(client.clone(), "spending_rate".to_string()), + utxo_turnover_1y: SeriesPattern1::new(client.clone(), "utxo_turnover_1y".to_string()), } } } @@ -8197,7 +8642,7 @@ impl SeriesTree_Cohorts_Utxo_All_Unrealized_Sentiment { /// Series tree node. pub struct SeriesTree_Cohorts_Utxo_Sth { pub supply: DeltaDominanceHalfInTotalPattern2, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub activity: CoindaysCoinyearsDormancyTransferPattern, pub realized: SeriesTree_Cohorts_Utxo_Sth_Realized, pub cost_basis: InMaxMinPerSupplyPattern, @@ -8209,7 +8654,7 @@ impl SeriesTree_Cohorts_Utxo_Sth { pub fn new(client: Arc, base_path: String) -> Self { Self { supply: DeltaDominanceHalfInTotalPattern2::new(client.clone(), "sth_supply".to_string()), - outputs: SpendingSpentUnspentPattern::new(client.clone(), "sth".to_string()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), "sth".to_string()), activity: CoindaysCoinyearsDormancyTransferPattern::new(client.clone(), "sth".to_string()), realized: SeriesTree_Cohorts_Utxo_Sth_Realized::new(client.clone(), format!("{base_path}_realized")), cost_basis: InMaxMinPerSupplyPattern::new(client.clone(), "sth".to_string()), @@ -8467,7 +8912,7 @@ impl SeriesTree_Cohorts_Utxo_Sth_Realized_Price_StdDev_1y { /// Series tree node. pub struct SeriesTree_Cohorts_Utxo_Lth { pub supply: DeltaDominanceHalfInTotalPattern2, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub activity: CoindaysCoinyearsDormancyTransferPattern, pub realized: SeriesTree_Cohorts_Utxo_Lth_Realized, pub cost_basis: InMaxMinPerSupplyPattern, @@ -8479,7 +8924,7 @@ impl SeriesTree_Cohorts_Utxo_Lth { pub fn new(client: Arc, base_path: String) -> Self { Self { supply: DeltaDominanceHalfInTotalPattern2::new(client.clone(), "lth_supply".to_string()), - outputs: SpendingSpentUnspentPattern::new(client.clone(), "lth".to_string()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), "lth".to_string()), activity: CoindaysCoinyearsDormancyTransferPattern::new(client.clone(), "lth".to_string()), realized: SeriesTree_Cohorts_Utxo_Lth_Realized::new(client.clone(), format!("{base_path}_realized")), cost_basis: InMaxMinPerSupplyPattern::new(client.clone(), "lth".to_string()), @@ -8967,7 +9412,7 @@ impl SeriesTree_Cohorts_Utxo_Entry { /// Series tree node. pub struct SeriesTree_Cohorts_Utxo_Entry_Discount { pub supply: DeltaDominanceHalfInTotalPattern2, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub activity: CoindaysCoinyearsDormancyTransferPattern, pub realized: SeriesTree_Cohorts_Utxo_Entry_Discount_Realized, pub cost_basis: InMaxMinPerSupplyPattern, @@ -8979,7 +9424,7 @@ impl SeriesTree_Cohorts_Utxo_Entry_Discount { pub fn new(client: Arc, base_path: String) -> Self { Self { supply: DeltaDominanceHalfInTotalPattern2::new(client.clone(), "veteran_supply".to_string()), - outputs: SpendingSpentUnspentPattern::new(client.clone(), "veteran".to_string()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), "veteran".to_string()), activity: CoindaysCoinyearsDormancyTransferPattern::new(client.clone(), "veteran".to_string()), realized: SeriesTree_Cohorts_Utxo_Entry_Discount_Realized::new(client.clone(), format!("{base_path}_realized")), cost_basis: InMaxMinPerSupplyPattern::new(client.clone(), "veteran".to_string()), @@ -9237,7 +9682,7 @@ impl SeriesTree_Cohorts_Utxo_Entry_Discount_Realized_Price_StdDev_1y { /// Series tree node. pub struct SeriesTree_Cohorts_Utxo_Entry_Premium { pub supply: DeltaDominanceHalfInTotalPattern2, - pub outputs: SpendingSpentUnspentPattern, + pub outputs: SpentUnspentUtxoPattern, pub activity: CoindaysCoinyearsDormancyTransferPattern, pub realized: SeriesTree_Cohorts_Utxo_Entry_Premium_Realized, pub cost_basis: InMaxMinPerSupplyPattern, @@ -9249,7 +9694,7 @@ impl SeriesTree_Cohorts_Utxo_Entry_Premium { pub fn new(client: Arc, base_path: String) -> Self { Self { supply: DeltaDominanceHalfInTotalPattern2::new(client.clone(), "rookie_supply".to_string()), - outputs: SpendingSpentUnspentPattern::new(client.clone(), "rookie".to_string()), + outputs: SpentUnspentUtxoPattern::new(client.clone(), "rookie".to_string()), activity: CoindaysCoinyearsDormancyTransferPattern::new(client.clone(), "rookie".to_string()), realized: SeriesTree_Cohorts_Utxo_Entry_Premium_Realized::new(client.clone(), format!("{base_path}_realized")), cost_basis: InMaxMinPerSupplyPattern::new(client.clone(), "rookie".to_string()), diff --git a/crates/brk_computer/src/cointime/activity/compute.rs b/crates/brk_computer/src/cointime/activity/compute.rs deleted file mode 100644 index 4cdee08d6..000000000 --- a/crates/brk_computer/src/cointime/activity/compute.rs +++ /dev/null @@ -1,58 +0,0 @@ -use brk_error::Result; -use brk_indexer::Indexer; -use brk_types::{Bitcoin, StoredF64}; -use vecdb::Exit; - -use super::Vecs; -use crate::distribution; - -impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - distribution: &distribution::Vecs, - exit: &Exit, - ) -> Result<()> { - let starting_height = indexer.safe_lengths().height; - let all_metrics = &distribution.utxo_cohorts.all.metrics; - let circulating_supply = &all_metrics.supply.total.sats.height; - - self.coinblocks_created - .compute(starting_height, exit, |vec| { - vec.compute_transform( - starting_height, - circulating_supply, - |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), - exit, - )?; - Ok(()) - })?; - - self.coinblocks_stored - .compute(starting_height, exit, |vec| { - vec.compute_subtract( - starting_height, - &self.coinblocks_created.block, - &distribution.coinblocks_destroyed.block, - exit, - )?; - Ok(()) - })?; - - self.liveliness.height.compute_divide( - starting_height, - &distribution.coinblocks_destroyed.cumulative.height, - &self.coinblocks_created.cumulative.height, - exit, - )?; - - self.ratio.height.compute_divide( - starting_height, - &self.liveliness.height, - &self.vaultedness.height, - exit, - )?; - - Ok(()) - } -} diff --git a/crates/brk_computer/src/cointime/supply/compute.rs b/crates/brk_computer/src/cointime/supply/compute.rs deleted file mode 100644 index 6a37486b8..000000000 --- a/crates/brk_computer/src/cointime/supply/compute.rs +++ /dev/null @@ -1,47 +0,0 @@ -use brk_error::Result; -use brk_indexer::Indexer; -use vecdb::Exit; - -use super::super::activity; -use super::Vecs; -use crate::{distribution, price}; - -impl Vecs { - pub(crate) fn compute( - &mut self, - indexer: &Indexer, - prices: &price::Vecs, - distribution: &distribution::Vecs, - activity: &activity::Vecs, - exit: &Exit, - ) -> Result<()> { - let starting_height = indexer.safe_lengths().height; - let circulating_supply = &distribution - .utxo_cohorts - .all - .metrics - .supply - .total - .sats - .height; - - self.vaulted.sats.height.compute_multiply( - starting_height, - circulating_supply, - &activity.vaultedness.height, - exit, - )?; - - self.active.sats.height.compute_multiply( - starting_height, - circulating_supply, - &activity.liveliness.height, - exit, - )?; - - self.vaulted.compute(prices, starting_height, exit)?; - self.active.compute(prices, starting_height, exit)?; - - Ok(()) - } -} diff --git a/crates/brk_computer/src/cointime/supply/import.rs b/crates/brk_computer/src/cointime/supply/import.rs deleted file mode 100644 index ef253e086..000000000 --- a/crates/brk_computer/src/cointime/supply/import.rs +++ /dev/null @@ -1,19 +0,0 @@ -use brk_error::Result; -use brk_types::Version; -use vecdb::Database; - -use super::Vecs; -use crate::{indexes, internal::ValuePerBlock}; - -impl Vecs { - pub(crate) fn forced_import( - db: &Database, - version: Version, - indexes: &indexes::Vecs, - ) -> Result { - Ok(Self { - vaulted: ValuePerBlock::forced_import(db, "vaulted_supply", version, indexes)?, - active: ValuePerBlock::forced_import(db, "active_supply", version, indexes)?, - }) - } -} diff --git a/crates/brk_computer/src/cointime/supply/vecs.rs b/crates/brk_computer/src/cointime/supply/vecs.rs deleted file mode 100644 index 3381435ba..000000000 --- a/crates/brk_computer/src/cointime/supply/vecs.rs +++ /dev/null @@ -1,10 +0,0 @@ -use brk_traversable::Traversable; -use vecdb::{Rw, StorageMode}; - -use crate::internal::ValuePerBlock; - -#[derive(Traversable)] -pub struct Vecs { - pub vaulted: ValuePerBlock, - pub active: ValuePerBlock, -} diff --git a/crates/brk_computer/src/distribution/addr/supply/share.rs b/crates/brk_computer/src/distribution/addr/supply/share.rs index f56f88307..0df72883c 100644 --- a/crates/brk_computer/src/distribution/addr/supply/share.rs +++ b/crates/brk_computer/src/distribution/addr/supply/share.rs @@ -48,11 +48,11 @@ impl AddrSupplyShareVecs { ) -> Result<()> { self.all .compute_binary::>( - max_from, - &supply.all.sats.height, - all_supply_sats, - exit, - )?; + max_from, + &supply.all.sats.height, + all_supply_sats, + exit, + )?; for ((_, share), ((_, cat), (_, denom))) in self .by_addr_type .iter_mut() diff --git a/crates/brk_computer/src/distribution/metrics/outputs/base.rs b/crates/brk_computer/src/distribution/metrics/outputs/base.rs index 30f52674d..f98a0f4e6 100644 --- a/crates/brk_computer/src/distribution/metrics/outputs/base.rs +++ b/crates/brk_computer/src/distribution/metrics/outputs/base.rs @@ -19,7 +19,7 @@ use crate::{ pub struct OutputsBase { pub unspent_count: PerBlockWithDeltas, pub spent_count: PerBlockCumulativeRolling, - pub spending_rate: PerBlock, + pub utxo_turnover_1y: PerBlock, } impl OutputsBase { @@ -35,7 +35,7 @@ impl OutputsBase { cfg.cached_starts, )?, spent_count: cfg.import("spent_utxo_count", v1)?, - spending_rate: cfg.import("spending_rate", Version::TWO)?, + utxo_turnover_1y: cfg.import("utxo_turnover_1y", Version::TWO)?, }) } @@ -73,7 +73,7 @@ impl OutputsBase { all_utxo_count: &impl ReadableVec, exit: &Exit, ) -> Result<()> { - self.spending_rate + self.utxo_turnover_1y .compute_binary::( max_from, &self.spent_count.sum.0._1y.height, diff --git a/crates/brk_computer/src/frameworks/coinflow/compute.rs b/crates/brk_computer/src/frameworks/coinflow/compute.rs new file mode 100644 index 000000000..43ee0de36 --- /dev/null +++ b/crates/brk_computer/src/frameworks/coinflow/compute.rs @@ -0,0 +1,562 @@ +use brk_error::Result; +use brk_indexer::{Indexer, Lengths}; +use brk_types::{Bitcoin, Cents, Height, Sats, StoredF64, Timestamp, Version}; +use vecdb::{AnyStoredVec, Exit, ReadableVec, WritableVec}; + +use super::super::cointime; +use super::{ + AGE_COHORT_COUNT, AgeBand, CohortVecs, HORIZON_COUNT, Horizons, MINIMUM_DURATION_DAYS, Vecs, + age_bounds_days, horizon_mobility, mobility, +}; +use crate::{ + distribution, frameworks::WeightedRatio, indexes, + internal::db_utils::validate_any_computed_version_or_reset, price, +}; + +const WRITE_INTERVAL: usize = 10_000; + +#[derive(Clone, Copy)] +struct DecayFit { + slope: f64, + tau: f64, + anchor_age: f64, + anchor_hazard: f64, +} + +impl Vecs { + pub(crate) fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + prices: &price::Vecs, + distribution: &distribution::Vecs, + cointime: &cointime::Vecs, + exit: &Exit, + ) -> Result<()> { + self.db.sync_bg_tasks()?; + + let starting_lengths = indexer.safe_lengths(); + let source_cohorts: Vec<_> = distribution.utxo_cohorts.age_range.iter().collect(); + let transfer_volumes: Vec<_> = source_cohorts + .iter() + .map(|cohort| { + &cohort + .metrics + .activity + .transfer_volume + .cumulative + .sats + .height + }) + .collect(); + let supplies: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.supply.total.sats.height) + .collect(); + let loss_supplies: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.supply.in_loss.sats.height) + .collect(); + let realized_caps: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.realized.cap.cents.height) + .collect(); + let coindays_created: Vec<_> = cointime + .age_range + .iter() + .map(|cohort| &cohort.coindays_created.cumulative.height) + .collect(); + + let rest_start = self.compute_primary( + &starting_lengths, + &indexes.timestamp.monotonic, + &transfer_volumes, + &coindays_created, + &supplies, + &loss_supplies, + &realized_caps, + exit, + )?; + let mut rest_lengths = starting_lengths.clone(); + rest_lengths.height = rest_start; + self.compute_rest(prices, &rest_lengths, exit)?; + + let exit = exit.clone(); + self.db.run_bg(move |db| { + let _lock = exit.lock(); + db.compact_deferred_default() + }); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn compute_primary( + &mut self, + starting_lengths: &Lengths, + timestamps: &T, + transfer_volumes: &[&S], + coindays_created: &[&D], + supplies: &[&S], + loss_supplies: &[&S], + realized_caps: &[&C], + exit: &Exit, + ) -> Result + where + T: ReadableVec, + D: ReadableVec, + S: ReadableVec, + C: ReadableVec, + { + debug_assert_eq!(transfer_volumes.len(), AGE_COHORT_COUNT); + debug_assert_eq!(coindays_created.len(), AGE_COHORT_COUNT); + debug_assert_eq!(supplies.len(), AGE_COHORT_COUNT); + debug_assert_eq!(loss_supplies.len(), AGE_COHORT_COUNT); + debug_assert_eq!(realized_caps.len(), AGE_COHORT_COUNT); + + let source_version: Version = std::iter::once(timestamps.version()) + .chain(transfer_volumes.iter().map(|vec| vec.version())) + .chain(coindays_created.iter().map(|vec| vec.version())) + .chain(supplies.iter().map(|vec| vec.version())) + .chain(loss_supplies.iter().map(|vec| vec.version())) + .chain(realized_caps.iter().map(|vec| vec.version())) + .sum(); + + for vec in self.primary_vecs_mut() { + validate_any_computed_version_or_reset(vec, source_version)?; + } + + let start = self + .primary_vecs_mut() + .into_iter() + .map(|vec| vec.len()) + .min() + .unwrap_or_default() + .min(usize::from(starting_lengths.height)); + + for vec in self.primary_vecs_mut() { + vec.any_truncate_if_needed_at(start)?; + } + + let source_end = transfer_volumes + .iter() + .map(|vec| vec.len()) + .chain(coindays_created.iter().map(|vec| vec.len())) + .chain(supplies.iter().map(|vec| vec.len())) + .chain(loss_supplies.iter().map(|vec| vec.len())) + .chain(realized_caps.iter().map(|vec| vec.len())) + .chain(std::iter::once(timestamps.len())) + .min() + .unwrap_or_default(); + + if source_end == 0 { + return Ok(Height::ZERO); + } + + let genesis_timestamp = timestamps + .collect_one(Height::ZERO) + .unwrap_or(Timestamp::ZERO); + let bounds = age_bounds_days(); + + let mut chunk_start = start; + while chunk_start < source_end { + let chunk_end = (chunk_start + WRITE_INTERVAL).min(source_end); + let timestamp_batch = timestamps.collect_range_at(chunk_start, chunk_end); + let transfer_batches: Vec<_> = transfer_volumes + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + let coinday_batches: Vec<_> = coindays_created + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + let supply_batches: Vec<_> = supplies + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + let loss_supply_batches: Vec<_> = loss_supplies + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + let cap_batches: Vec<_> = realized_caps + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + + for offset in 0..(chunk_end - chunk_start) { + let hazards = std::array::from_fn(|index| { + spending_rate( + transfer_batches[index][offset], + coinday_batches[index][offset], + ) + }); + let network_age = timestamp_batch[offset] + .difference_in_days_between_float(genesis_timestamp) + .max(MINIMUM_DURATION_DAYS); + let exposures = spending_exposures(&hazards, network_age, &bounds); + let mobilities = exposures.map(mobility); + let horizon_mobilities: Horizons<[f64; AGE_COHORT_COUNT]> = + Horizons::from_fn(|_, horizon| { + std::array::from_fn(|age| horizon_mobility(&hazards, age, horizon, &bounds)) + }); + + let mut mobile_supply = Sats::ZERO; + let mut immobile_supply = Sats::ZERO; + let mut coinflow_cap = Cents::ZERO; + let mut supply_in_loss = WeightedRatio::default(); + let mut horizon_supply_in_loss = Horizons::from_fn(|_, _| WeightedRatio::default()); + + for (index, cohort) in self.age_range.iter_mut().enumerate() { + let mobility = StoredF64::from(mobilities[index]); + let total_supply = supply_batches[index][offset]; + let cohort_mobile_supply = total_supply * mobility; + let cohort_immobile_supply = total_supply - cohort_mobile_supply; + + let total_cap = cap_batches[index][offset]; + let cohort_mobile_cap = total_cap * mobility; + let total = total_supply.as_u128() as f64; + let loss = loss_supply_batches[index][offset].as_u128() as f64; + supply_in_loss.add(loss, total, mobilities[index]); + for (ratio, weights) in horizon_supply_in_loss + .iter_mut() + .zip(horizon_mobilities.iter()) + { + ratio.add(loss, total, weights[index]); + } + + cohort + .spending_rate + .height + .push(StoredF64::from(hazards[index])); + cohort + .spending_exposure + .height + .push(StoredF64::from(exposures[index])); + cohort.mobility.height.push(mobility); + cohort.supply.mobile.sats.height.push(cohort_mobile_supply); + cohort + .supply + .immobile + .sats + .height + .push(cohort_immobile_supply); + + mobile_supply += cohort_mobile_supply; + immobile_supply += cohort_immobile_supply; + coinflow_cap += cohort_mobile_cap; + } + + self.supply.mobile.sats.height.push(mobile_supply); + self.supply.immobile.sats.height.push(immobile_supply); + self.supply_in_loss_share + .height + .push(supply_in_loss.value()); + for (output, ratio) in self.horizon.iter_mut().zip(horizon_supply_in_loss.iter()) { + output.supply_in_loss_share.height.push(ratio.value()); + } + self.cap.cents.height.push(coinflow_cap); + self.price + .cents + .height + .push(realized_price(coinflow_cap, mobile_supply)); + } + + { + let _lock = exit.lock(); + for vec in self.primary_vecs_mut() { + vec.write()?; + } + } + chunk_start = chunk_end; + } + + Ok(Height::from(start)) + } + + fn compute_rest( + &mut self, + prices: &price::Vecs, + starting_lengths: &Lengths, + exit: &Exit, + ) -> Result<()> { + for cohort in self.age_range.iter_mut() { + cohort + .supply + .mobile + .compute(prices, starting_lengths.height, exit)?; + cohort + .supply + .immobile + .compute(prices, starting_lengths.height, exit)?; + } + + self.supply + .mobile + .compute(prices, starting_lengths.height, exit)?; + self.supply + .immobile + .compute(prices, starting_lengths.height, exit)?; + self.price + .compute_ratio(starting_lengths, &prices.spot.cents.height, exit)?; + Ok(()) + } + + fn primary_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> { + let mut vecs = Vec::with_capacity(AGE_COHORT_COUNT * 5 + 5 + HORIZON_COUNT); + + for cohort in self.age_range.iter_mut() { + vecs.extend(cohort.primary_vecs_mut()); + } + + vecs.extend([ + &mut self.supply.mobile.sats.height as &mut dyn AnyStoredVec, + &mut self.supply.immobile.sats.height, + &mut self.supply_in_loss_share.height, + &mut self.cap.cents.height, + &mut self.price.cents.height, + ]); + vecs.extend( + self.horizon + .iter_mut() + .map(|horizon| &mut horizon.supply_in_loss_share.height as &mut dyn AnyStoredVec), + ); + vecs + } +} + +impl CohortVecs { + fn primary_vecs_mut(&mut self) -> [&mut dyn AnyStoredVec; 5] { + [ + &mut self.spending_rate.height, + &mut self.spending_exposure.height, + &mut self.mobility.height, + &mut self.supply.mobile.sats.height, + &mut self.supply.immobile.sats.height, + ] + } +} + +#[inline] +fn spending_rate(transfer_volume: Sats, coindays_created: StoredF64) -> f64 { + let exposure = f64::from(coindays_created); + if exposure > 0.0 { + (f64::from(Bitcoin::from(transfer_volume)) / exposure).max(0.0) + } else { + 0.0 + } +} + +fn fit_decay( + hazards: &[f64; AGE_COHORT_COUNT], + network_age: f64, + bounds: &[AgeBand; AGE_COHORT_COUNT], +) -> Option { + let mut total_duration = 0.0; + let mut weighted_age = 0.0; + let mut weighted_log_hazard = 0.0; + let mut anchor = None; + + for (index, band) in bounds[..AGE_COHORT_COUNT - 1].iter().enumerate() { + let hazard = hazards[index]; + if band.upper > network_age || !hazard.is_finite() || hazard <= 0.0 { + continue; + } + + let age = (band.lower + band.upper) / 2.0; + let duration = band.upper - band.lower; + let log_hazard = hazard.ln(); + total_duration += duration; + weighted_age += duration * age; + weighted_log_hazard += duration * log_hazard; + anchor = Some((band.upper, hazard)); + } + + if total_duration <= 0.0 { + return None; + } + + let mean_age = weighted_age / total_duration; + let mean_log_hazard = weighted_log_hazard / total_duration; + let mut covariance = 0.0; + let mut age_variance = 0.0; + + for (index, band) in bounds[..AGE_COHORT_COUNT - 1].iter().enumerate() { + let hazard = hazards[index]; + if band.upper > network_age || !hazard.is_finite() || hazard <= 0.0 { + continue; + } + + let age = (band.lower + band.upper) / 2.0; + let duration = band.upper - band.lower; + let log_hazard = hazard.ln(); + let age_offset = age - mean_age; + covariance += duration * age_offset * (log_hazard - mean_log_hazard); + age_variance += duration * age_offset.powi(2); + } + + if age_variance <= f64::EPSILON { + return None; + } + + let slope = covariance / age_variance; + if slope >= 0.0 { + return None; + } + + let tau = -1.0 / slope; + if !tau.is_finite() || tau <= 0.0 { + return None; + } + + let (anchor_age, anchor_hazard) = anchor?; + Some(DecayFit { + slope, + tau, + anchor_age, + anchor_hazard, + }) +} + +fn spending_exposures( + hazards: &[f64; AGE_COHORT_COUNT], + network_age: f64, + bounds: &[AgeBand; AGE_COHORT_COUNT], +) -> [f64; AGE_COHORT_COUNT] { + let Some(fit) = fit_decay(hazards, network_age, bounds) else { + return [0.0; AGE_COHORT_COUNT]; + }; + + std::array::from_fn(|start_band| { + spending_exposure(hazards, start_band, network_age, bounds, fit) + }) +} + +fn spending_exposure( + hazards: &[f64; AGE_COHORT_COUNT], + start_band: usize, + network_age: f64, + bounds: &[AgeBand; AGE_COHORT_COUNT], + fit: DecayFit, +) -> f64 { + let start = bounds[start_band]; + let occupied_upper = if start.upper.is_finite() { + start.upper.min(network_age.max(start.lower)) + } else { + start.lower + }; + let mut age = if start.upper.is_finite() { + (start.lower + occupied_upper) / 2.0 + } else { + start.lower + }; + let mut exposure = 0.0; + + for band_index in start_band..AGE_COHORT_COUNT - 1 { + let band = bounds[band_index]; + let duration = (band.upper - age.max(band.lower)).max(MINIMUM_DURATION_DAYS); + let hazard = hazards[band_index]; + let observed = band.upper <= network_age && hazard.is_finite() && hazard > 0.0; + if !observed { + break; + } + + exposure += hazard * duration; + age = band.upper; + } + + let tail = bounds[AGE_COHORT_COUNT - 1]; + let tail_hazard = hazards[AGE_COHORT_COUNT - 1]; + let observed_tail = network_age > tail.lower && tail_hazard.is_finite() && tail_hazard > 0.0; + let (anchor_age, anchor_hazard) = if observed_tail { + (tail.lower, tail_hazard) + } else { + (fit.anchor_age, fit.anchor_hazard) + }; + let continuation_age = age.max(anchor_age); + let continuation_hazard = anchor_hazard * (fit.slope * (continuation_age - anchor_age)).exp(); + exposure + (continuation_hazard * fit.tau).max(0.0) +} + +#[inline] +fn realized_price(cap: Cents, supply: Sats) -> Cents { + (cap.as_u128() * Sats::ONE_BTC_U128) + .checked_div(supply.as_u128()) + .map(Cents::from) + .unwrap_or(Cents::ZERO) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mobility_is_the_complement_of_survival() { + assert_eq!(mobility(0.0), 0.0); + assert!((mobility(2.0_f64.ln()) - 0.5).abs() < 1e-12); + assert!((mobility(1e-15) - 1e-15).abs() < 1e-27); + assert!(mobility(1_000.0) < 1.0); + assert_eq!(mobility(f64::INFINITY), 1.0 - 1e-12); + assert_eq!(mobility(f64::NAN), 0.0); + } + + #[test] + fn fixed_horizon_compounds_hazards_across_age_ranges() { + let bounds = age_bounds_days(); + let hazards = [0.01; AGE_COHORT_COUNT]; + let probability = horizon_mobility(&hazards, 2, 30.0, &bounds); + + assert!((probability - mobility(0.3)).abs() < 1e-12); + } + + #[test] + fn decay_fit_recovers_an_exponential_lifetime() { + let bounds = age_bounds_days(); + let expected_tau = 1_000.0; + let hazards = std::array::from_fn(|index| { + let band = bounds[index]; + let age = if band.upper.is_finite() { + (band.lower + band.upper) / 2.0 + } else { + band.lower + }; + (-age / expected_tau).exp() + }); + + let fit = fit_decay(&hazards, 20.0 * 365.0, &bounds).unwrap(); + + assert!((fit.tau - expected_tau).abs() < 1e-9); + } + + #[test] + fn oldest_cohort_exposure_is_its_observed_tail_lifetime() { + let bounds = age_bounds_days(); + let hazards = std::array::from_fn(|index| { + let band = bounds[index]; + let age = if band.upper.is_finite() { + (band.lower + band.upper) / 2.0 + } else { + band.lower + }; + (-age / 1_000.0).exp() + }); + let network_age = 20.0 * 365.0; + let fit = fit_decay(&hazards, network_age, &bounds).unwrap(); + let exposures = spending_exposures(&hazards, network_age, &bounds); + + assert!( + (exposures[AGE_COHORT_COUNT - 1] - hazards[AGE_COHORT_COUNT - 1] * fit.tau).abs() + < 1e-12 + ); + } + + #[test] + fn supply_partitions_and_coinflow_cap_is_bounded() { + let total_supply = Sats::from(123_456_789_u64); + let total_cap = Cents::from(987_654_321_u64); + let mobility = StoredF64::from(0.321); + let mobile_supply = total_supply * mobility; + let coinflow_cap = total_cap * mobility; + + assert_eq!(mobile_supply + (total_supply - mobile_supply), total_supply); + assert!(coinflow_cap <= total_cap); + } +} diff --git a/crates/brk_computer/src/frameworks/coinflow/import.rs b/crates/brk_computer/src/frameworks/coinflow/import.rs new file mode 100644 index 000000000..a651838a1 --- /dev/null +++ b/crates/brk_computer/src/frameworks/coinflow/import.rs @@ -0,0 +1,94 @@ +use std::path::Path; + +use brk_cohort::{AgeRange, CohortContext}; +use brk_error::Result; +use brk_types::Version; + +use super::{CohortVecs, DB_NAME, HorizonVecs, Horizons, Split, Vecs}; +use crate::{ + indexes, + internal::{ + FiatPerBlock, PerBlock, PriceWithRatioPerBlock, ValuePerBlock, + db_utils::{finalize_db, open_db}, + }, +}; + +const VERSION: Version = Version::TWO; + +fn import_split(mut import: impl FnMut(&str) -> Result) -> Result> { + Ok(Split { + mobile: import("mobile")?, + immobile: import("immobile")?, + }) +} + +impl CohortVecs { + fn forced_import( + db: &vecdb::Database, + name: &str, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + Ok(Self { + spending_rate: PerBlock::forced_import( + db, + &format!("{name}_spending_rate"), + version, + indexes, + )?, + spending_exposure: PerBlock::forced_import( + db, + &format!("{name}_spending_exposure"), + version, + indexes, + )?, + mobility: PerBlock::forced_import(db, &format!("{name}_mobility"), version, indexes)?, + supply: import_split(|side| { + ValuePerBlock::forced_import(db, &format!("{name}_{side}_supply"), version, indexes) + })?, + }) + } +} + +impl Vecs { + pub(crate) fn forced_import( + parent_path: &Path, + parent_version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let db = open_db(parent_path, DB_NAME, 250_000)?; + let version = parent_version + VERSION; + + let this = Self { + age_range: AgeRange::try_new(|_, name| { + let name = CohortContext::Utxo.prefixed(name); + CohortVecs::forced_import(&db, &name, version, indexes) + })?, + supply: import_split(|side| { + ValuePerBlock::forced_import(&db, &format!("{side}_supply"), version, indexes) + })?, + supply_in_loss_share: PerBlock::forced_import( + &db, + "coinflow_supply_in_loss_share", + version, + indexes, + )?, + horizon: Horizons::try_from_fn(|horizon, _| -> Result<_> { + Ok(HorizonVecs { + supply_in_loss_share: PerBlock::forced_import( + &db, + &format!("coinflow_{horizon}_supply_in_loss_share"), + version, + indexes, + )?, + }) + })?, + cap: FiatPerBlock::forced_import(&db, "coinflow_cap", version, indexes)?, + price: PriceWithRatioPerBlock::forced_import(&db, "coinflow_price", version, indexes)?, + db, + }; + + finalize_db(&this.db, &this)?; + Ok(this) + } +} diff --git a/crates/brk_computer/src/frameworks/coinflow/mod.rs b/crates/brk_computer/src/frameworks/coinflow/mod.rs new file mode 100644 index 000000000..fd8a74562 --- /dev/null +++ b/crates/brk_computer/src/frameworks/coinflow/mod.rs @@ -0,0 +1,79 @@ +mod compute; +mod import; +mod vecs; + +use brk_cohort::AGE_RANGE_BOUNDS; + +pub(crate) use vecs::HORIZON_DAYS; +pub use vecs::{CohortVecs, HorizonVecs, Horizons, Split, Vecs}; + +pub const DB_NAME: &str = "coinflow"; + +pub(crate) const AGE_COHORT_COUNT: usize = 21; +pub(crate) const HORIZON_COUNT: usize = 7; +pub(crate) const HOURS_PER_DAY: f64 = 24.0; +pub(crate) const MINIMUM_DURATION_DAYS: f64 = 1.0 / HOURS_PER_DAY; + +#[derive(Clone, Copy)] +pub(crate) struct AgeBand { + pub lower: f64, + pub upper: f64, +} + +pub(crate) fn age_bounds_days() -> [AgeBand; AGE_COHORT_COUNT] { + let mut bounds = AGE_RANGE_BOUNDS.iter(); + std::array::from_fn(|index| { + let bound = bounds.next().unwrap(); + AgeBand { + lower: bound.start as f64 / HOURS_PER_DAY, + upper: if index + 1 < AGE_COHORT_COUNT { + bound.end as f64 / HOURS_PER_DAY + } else { + f64::INFINITY + }, + } + }) +} + +#[inline] +pub(crate) fn mobility(exposure: f64) -> f64 { + if exposure.is_nan() || exposure <= 0.0 { + 0.0 + } else { + (-(-exposure).exp_m1()).min(1.0 - 1e-12) + } +} + +pub(crate) fn horizon_mobility( + hazards: &[f64; AGE_COHORT_COUNT], + start_band: usize, + horizon: f64, + bounds: &[AgeBand; AGE_COHORT_COUNT], +) -> f64 { + let start = bounds[start_band]; + let mut age = if start.upper.is_finite() { + (start.lower + start.upper) / 2.0 + } else { + start.lower + }; + let mut remaining = horizon; + let mut band = start_band; + let mut exposure = 0.0; + + while remaining > 0.0 && band < AGE_COHORT_COUNT { + let upper = bounds[band].upper; + let duration = if upper.is_finite() { + remaining.min((upper - age).max(MINIMUM_DURATION_DAYS)) + } else { + remaining + }; + exposure += hazards[band].max(0.0) * duration; + remaining -= duration; + band += 1; + if band < AGE_COHORT_COUNT { + age = bounds[band].lower; + } + } + + mobility(exposure) +} diff --git a/crates/brk_computer/src/frameworks/coinflow/vecs.rs b/crates/brk_computer/src/frameworks/coinflow/vecs.rs new file mode 100644 index 000000000..d0631ae86 --- /dev/null +++ b/crates/brk_computer/src/frameworks/coinflow/vecs.rs @@ -0,0 +1,129 @@ +use brk_cohort::AgeRange; +use brk_traversable::Traversable; +use brk_types::{Cents, StoredF64}; +use vecdb::{Database, Rw, StorageMode}; + +use crate::internal::{FiatPerBlock, PerBlock, PriceWithRatioPerBlock, ValuePerBlock}; + +#[derive(Clone, Copy, Traversable)] +pub struct Horizons { + pub _8y: T, + pub _4y: T, + pub _2y: T, + pub _1y: T, + pub _6m: T, + pub _3m: T, + pub _1m: T, +} + +pub(crate) const HORIZON_NAMES: Horizons<&str> = Horizons { + _8y: "8y", + _4y: "4y", + _2y: "2y", + _1y: "1y", + _6m: "6m", + _3m: "3m", + _1m: "1m", +}; + +pub(crate) const HORIZON_DAYS: Horizons = Horizons { + _8y: 8.0 * 365.0, + _4y: 4.0 * 365.0, + _2y: 2.0 * 365.0, + _1y: 365.0, + _6m: 180.0, + _3m: 90.0, + _1m: 30.0, +}; + +impl Horizons { + pub(crate) fn from_fn(mut create: impl FnMut(&'static str, f64) -> T) -> Self { + let names = HORIZON_NAMES; + let days = HORIZON_DAYS; + Self { + _8y: create(names._8y, days._8y), + _4y: create(names._4y, days._4y), + _2y: create(names._2y, days._2y), + _1y: create(names._1y, days._1y), + _6m: create(names._6m, days._6m), + _3m: create(names._3m, days._3m), + _1m: create(names._1m, days._1m), + } + } + + pub(crate) fn try_from_fn( + mut create: impl FnMut(&'static str, f64) -> Result, + ) -> Result { + let names = HORIZON_NAMES; + let days = HORIZON_DAYS; + Ok(Self { + _8y: create(names._8y, days._8y)?, + _4y: create(names._4y, days._4y)?, + _2y: create(names._2y, days._2y)?, + _1y: create(names._1y, days._1y)?, + _6m: create(names._6m, days._6m)?, + _3m: create(names._3m, days._3m)?, + _1m: create(names._1m, days._1m)?, + }) + } + + pub(crate) fn as_array(&self) -> [&T; 7] { + [ + &self._8y, &self._4y, &self._2y, &self._1y, &self._6m, &self._3m, &self._1m, + ] + } + + pub(crate) fn as_mut_array(&mut self) -> [&mut T; 7] { + [ + &mut self._8y, + &mut self._4y, + &mut self._2y, + &mut self._1y, + &mut self._6m, + &mut self._3m, + &mut self._1m, + ] + } + + pub(crate) fn iter(&self) -> impl Iterator { + self.as_array().into_iter() + } + + pub(crate) fn iter_mut(&mut self) -> impl Iterator { + self.as_mut_array().into_iter() + } +} + +#[derive(Traversable)] +pub struct HorizonVecs { + #[traversable(wrap = "supply/in_loss", rename = "share")] + pub supply_in_loss_share: PerBlock, +} + +#[derive(Traversable)] +pub struct Split { + pub mobile: T, + pub immobile: T, +} + +#[derive(Traversable)] +pub struct CohortVecs { + pub spending_rate: PerBlock, + pub spending_exposure: PerBlock, + pub mobility: PerBlock, + pub supply: Split>, +} + +#[derive(Traversable)] +pub struct Vecs { + #[traversable(skip)] + pub(crate) db: Database, + + pub age_range: AgeRange>, + pub supply: Split>, + #[traversable(wrap = "supply/mobile/in_loss", rename = "share")] + pub supply_in_loss_share: PerBlock, + pub horizon: Horizons>, + pub cap: FiatPerBlock, + pub price: PriceWithRatioPerBlock, +} diff --git a/crates/brk_computer/src/frameworks/cointime/activity/compute.rs b/crates/brk_computer/src/frameworks/cointime/activity/compute.rs new file mode 100644 index 000000000..33490698f --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/activity/compute.rs @@ -0,0 +1,70 @@ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_types::{Bitcoin, StoredF64}; +use vecdb::Exit; + +use super::{DerivedVecs, Vecs}; +use crate::{distribution, internal::PerBlockCumulativeRolling}; + +pub(crate) fn compute_rest( + starting_height: brk_types::Height, + created: &PerBlockCumulativeRolling, + consumed: &PerBlockCumulativeRolling, + stored: &mut PerBlockCumulativeRolling, + derived: &mut DerivedVecs, + exit: &Exit, +) -> Result<()> { + stored.compute(starting_height, exit, |vec| { + vec.compute_subtract(starting_height, &created.block, &consumed.block, exit)?; + Ok(()) + })?; + + derived.liveliness.height.compute_divide( + starting_height, + &consumed.cumulative.height, + &created.cumulative.height, + exit, + )?; + + derived.ratio.height.compute_divide( + starting_height, + &derived.liveliness.height, + &derived.vaultedness.height, + exit, + )?; + + Ok(()) +} + +impl Vecs { + pub(crate) fn compute( + &mut self, + indexer: &Indexer, + distribution: &distribution::Vecs, + exit: &Exit, + ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + let all_metrics = &distribution.utxo_cohorts.all.metrics; + let circulating_supply = &all_metrics.supply.total.sats.height; + + self.coinblocks_created + .compute(starting_height, exit, |vec| { + vec.compute_transform( + starting_height, + circulating_supply, + |(i, v, ..)| (i, StoredF64::from(Bitcoin::from(v))), + exit, + )?; + Ok(()) + })?; + + compute_rest( + starting_height, + &self.coinblocks_created, + &distribution.coinblocks_destroyed, + &mut self.coinblocks_stored, + &mut self.derived, + exit, + ) + } +} diff --git a/crates/brk_computer/src/cointime/activity/import.rs b/crates/brk_computer/src/frameworks/cointime/activity/import.rs similarity index 58% rename from crates/brk_computer/src/cointime/activity/import.rs rename to crates/brk_computer/src/frameworks/cointime/activity/import.rs index 89dc42f7a..9c567231c 100644 --- a/crates/brk_computer/src/cointime/activity/import.rs +++ b/crates/brk_computer/src/frameworks/cointime/activity/import.rs @@ -2,7 +2,7 @@ use brk_error::Result; use brk_types::Version; use vecdb::{Database, ReadableCloneableVec}; -use super::Vecs; +use super::{DerivedVecs, Vecs}; use crate::{ indexes, internal::{ @@ -10,6 +10,37 @@ use crate::{ }, }; +impl DerivedVecs { + pub(crate) fn forced_import_with_prefix( + db: &Database, + prefix: &str, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let name = |metric: &str| { + if prefix.is_empty() { + metric.to_owned() + } else { + format!("{prefix}_{metric}") + } + }; + let liveliness_name = name("liveliness"); + let liveliness = PerBlock::forced_import(db, &liveliness_name, version, indexes)?; + let vaultedness = LazyPerBlock::from_computed::( + &name("vaultedness"), + version, + liveliness.height.read_only_boxed_clone(), + &liveliness, + ); + + Ok(Self { + liveliness, + vaultedness, + ratio: PerBlock::forced_import(db, &name("activity_to_vaultedness"), version, indexes)?, + }) + } +} + impl Vecs { pub(crate) fn forced_import( db: &Database, @@ -17,15 +48,6 @@ impl Vecs { indexes: &indexes::Vecs, cached_starts: &Windows<&WindowStartVec>, ) -> Result { - let liveliness = PerBlock::forced_import(db, "liveliness", version, indexes)?; - - let vaultedness = LazyPerBlock::from_computed::( - "vaultedness", - version, - liveliness.height.read_only_boxed_clone(), - &liveliness, - ); - Ok(Self { coinblocks_created: PerBlockCumulativeRolling::forced_import( db, @@ -41,9 +63,7 @@ impl Vecs { indexes, cached_starts, )?, - liveliness, - vaultedness, - ratio: PerBlock::forced_import(db, "activity_to_vaultedness", version, indexes)?, + derived: DerivedVecs::forced_import_with_prefix(db, "", version, indexes)?, }) } } diff --git a/crates/brk_computer/src/frameworks/cointime/activity/mod.rs b/crates/brk_computer/src/frameworks/cointime/activity/mod.rs new file mode 100644 index 000000000..7aea78f39 --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/activity/mod.rs @@ -0,0 +1,6 @@ +mod compute; +mod import; +mod vecs; + +pub(crate) use compute::compute_rest; +pub use vecs::{DerivedVecs, Vecs}; diff --git a/crates/brk_computer/src/cointime/activity/vecs.rs b/crates/brk_computer/src/frameworks/cointime/activity/vecs.rs similarity index 70% rename from crates/brk_computer/src/cointime/activity/vecs.rs rename to crates/brk_computer/src/frameworks/cointime/activity/vecs.rs index 5a77df1b2..aec5b5705 100644 --- a/crates/brk_computer/src/cointime/activity/vecs.rs +++ b/crates/brk_computer/src/frameworks/cointime/activity/vecs.rs @@ -1,14 +1,23 @@ use brk_traversable::Traversable; use brk_types::StoredF64; +use derive_more::{Deref, DerefMut}; use vecdb::{Rw, StorageMode}; use crate::internal::{LazyPerBlock, PerBlock, PerBlockCumulativeRolling}; #[derive(Traversable)] -pub struct Vecs { - pub coinblocks_created: PerBlockCumulativeRolling, - pub coinblocks_stored: PerBlockCumulativeRolling, +pub struct DerivedVecs { pub liveliness: PerBlock, pub vaultedness: LazyPerBlock, pub ratio: PerBlock, } + +#[derive(Deref, DerefMut, Traversable)] +pub struct Vecs { + pub coinblocks_created: PerBlockCumulativeRolling, + pub coinblocks_stored: PerBlockCumulativeRolling, + #[deref] + #[deref_mut] + #[traversable(flatten)] + pub derived: DerivedVecs, +} diff --git a/crates/brk_computer/src/cointime/adjusted/compute.rs b/crates/brk_computer/src/frameworks/cointime/adjusted/compute.rs similarity index 100% rename from crates/brk_computer/src/cointime/adjusted/compute.rs rename to crates/brk_computer/src/frameworks/cointime/adjusted/compute.rs diff --git a/crates/brk_computer/src/cointime/adjusted/import.rs b/crates/brk_computer/src/frameworks/cointime/adjusted/import.rs similarity index 100% rename from crates/brk_computer/src/cointime/adjusted/import.rs rename to crates/brk_computer/src/frameworks/cointime/adjusted/import.rs diff --git a/crates/brk_computer/src/cointime/activity/mod.rs b/crates/brk_computer/src/frameworks/cointime/adjusted/mod.rs similarity index 100% rename from crates/brk_computer/src/cointime/activity/mod.rs rename to crates/brk_computer/src/frameworks/cointime/adjusted/mod.rs diff --git a/crates/brk_computer/src/cointime/adjusted/vecs.rs b/crates/brk_computer/src/frameworks/cointime/adjusted/vecs.rs similarity index 100% rename from crates/brk_computer/src/cointime/adjusted/vecs.rs rename to crates/brk_computer/src/frameworks/cointime/adjusted/vecs.rs diff --git a/crates/brk_computer/src/frameworks/cointime/age_range/compute.rs b/crates/brk_computer/src/frameworks/cointime/age_range/compute.rs new file mode 100644 index 000000000..74fdef0da --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/age_range/compute.rs @@ -0,0 +1,402 @@ +use brk_cohort::AGE_RANGE_BOUNDS; +use brk_error::Result; +use brk_indexer::Indexer; +use brk_types::{Bitcoin, Height, ONE_DAY_IN_SEC_F64, Sats, StoredF64, Timestamp, Version}; +use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, WritableVec}; + +use super::super::activity; +use super::{CohortVecs, Vecs}; +use crate::{distribution, indexes, price}; + +const AGE_COHORT_COUNT: usize = 21; +const HOURS_PER_DAY: f64 = 24.0; +const WRITE_INTERVAL: usize = 10_000; + +impl Vecs { + pub(crate) fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + prices: &price::Vecs, + distribution: &distribution::Vecs, + exit: &Exit, + ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + let source_cohorts: Vec<_> = distribution.utxo_cohorts.age_range.iter().collect(); + let supplies: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.supply.total.sats.height) + .collect(); + let transfer_volumes: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.activity.transfer_volume.block.sats) + .collect(); + let coindays_destroyed: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.activity.coindays_destroyed.block) + .collect(); + + self.compute_created( + starting_height, + &indexes.timestamp.monotonic, + &supplies, + exit, + )?; + self.compute_consumed( + starting_height, + &transfer_volumes, + &coindays_destroyed, + exit, + )?; + self.compute_rest(starting_height, prices, &supplies, exit) + } + + fn compute_created( + &mut self, + starting_height: Height, + timestamps: &T, + supplies: &[&S], + exit: &Exit, + ) -> Result<()> + where + T: ReadableVec, + S: ReadableVec, + { + debug_assert_eq!(supplies.len(), AGE_COHORT_COUNT); + + let created_version: Version = std::iter::once(timestamps.version()) + .chain(supplies.iter().map(|vec| vec.version())) + .sum(); + let mut cohorts: Vec<&mut CohortVecs> = self.iter_mut().collect(); + + for cohort in cohorts.iter_mut() { + cohort + .coindays_created + .block + .validate_computed_version_or_reset(created_version)?; + } + + let start = cohorts + .iter() + .map(|cohort| cohort.coindays_created.block.len()) + .min() + .unwrap_or_default() + .min(usize::from(starting_height)); + + for cohort in cohorts.iter_mut() { + cohort.coindays_created.block.truncate_if_needed_at(start)?; + } + + let source_end = supplies + .iter() + .map(|vec| vec.len()) + .chain(std::iter::once(timestamps.len())) + .min() + .unwrap_or_default(); + + let mut chunk_start = start; + while chunk_start < source_end { + let chunk_end = (chunk_start + WRITE_INTERVAL).min(source_end); + let timestamp_start = chunk_start.saturating_sub(1); + let timestamp_batch = timestamps.collect_range_at(timestamp_start, chunk_end); + let supply_batches: Vec<_> = supplies + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + + for (offset, _) in supply_batches.first().unwrap().iter().enumerate() { + let interval_seconds = + monotonic_interval_seconds(×tamp_batch, chunk_start, offset); + + for (index, cohort) in cohorts.iter_mut().enumerate() { + cohort.coindays_created.block.push(coindays_created( + supply_batches[index][offset], + interval_seconds, + )); + } + } + + { + let _lock = exit.lock(); + for cohort in cohorts.iter_mut() { + cohort.coindays_created.block.write()?; + } + } + chunk_start = chunk_end; + } + + for cohort in cohorts { + cohort + .coindays_created + .compute_rest(starting_height, exit)?; + } + + Ok(()) + } + + fn compute_consumed( + &mut self, + starting_height: Height, + transfer_volumes: &[&V], + source_coindays_destroyed: &[&D], + exit: &Exit, + ) -> Result<()> + where + V: ReadableVec, + D: ReadableVec, + { + debug_assert_eq!(transfer_volumes.len(), AGE_COHORT_COUNT); + debug_assert_eq!(source_coindays_destroyed.len(), AGE_COHORT_COUNT); + + let destroyed_version: Version = transfer_volumes + .iter() + .map(|vec| vec.version()) + .chain(source_coindays_destroyed.iter().map(|vec| vec.version())) + .sum(); + let mut cohorts: Vec<&mut CohortVecs> = self.iter_mut().collect(); + + for cohort in cohorts.iter_mut() { + cohort + .coindays_consumed + .block + .validate_computed_version_or_reset(destroyed_version)?; + } + + let start = cohorts + .iter() + .map(|cohort| cohort.coindays_consumed.block.len()) + .min() + .unwrap_or_default() + .min(usize::from(starting_height)); + + for cohort in cohorts.iter_mut() { + cohort + .coindays_consumed + .block + .truncate_if_needed_at(start)?; + } + + let source_end = transfer_volumes + .iter() + .map(|vec| vec.len()) + .chain(source_coindays_destroyed.iter().map(|vec| vec.len())) + .min() + .unwrap_or_default(); + let bounds = age_bounds_days(); + + let mut chunk_start = start; + while chunk_start < source_end { + let chunk_end = (chunk_start + WRITE_INTERVAL).min(source_end); + let transfer_batches: Vec<_> = transfer_volumes + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + let destroyed_batches: Vec<_> = source_coindays_destroyed + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + + for offset in 0..(chunk_end - chunk_start) { + let volumes_btc: [f64; AGE_COHORT_COUNT] = std::array::from_fn(|index| { + f64::from(Bitcoin::from(transfer_batches[index][offset])) + }); + let cdd: [f64; AGE_COHORT_COUNT] = + std::array::from_fn(|index| f64::from(destroyed_batches[index][offset])); + let consumed = allocate_consumed_coindays(volumes_btc, cdd, &bounds); + + for (index, cohort) in cohorts.iter_mut().enumerate() { + cohort + .coindays_consumed + .block + .push(StoredF64::from(consumed[index])); + } + } + + { + let _lock = exit.lock(); + for cohort in cohorts.iter_mut() { + cohort.coindays_consumed.block.write()?; + } + } + chunk_start = chunk_end; + } + + for cohort in cohorts { + cohort + .coindays_consumed + .compute_rest(starting_height, exit)?; + } + + Ok(()) + } + + fn compute_rest( + &mut self, + starting_height: Height, + prices: &price::Vecs, + supplies: &[&S], + exit: &Exit, + ) -> Result<()> + where + S: ReadableVec, + { + debug_assert_eq!(supplies.len(), AGE_COHORT_COUNT); + + for (cohort, &total_supply) in self.iter_mut().zip(supplies) { + let CohortVecs { + coindays_created, + coindays_consumed, + coindays_stored, + activity: activity_vecs, + supply, + } = cohort; + + activity::compute_rest( + starting_height, + coindays_created, + coindays_consumed, + coindays_stored, + activity_vecs, + exit, + )?; + + supply.compute_from( + starting_height, + prices, + total_supply, + &activity_vecs.liveliness.height, + &activity_vecs.vaultedness.height, + exit, + )?; + } + + Ok(()) + } +} + +#[inline(always)] +fn monotonic_interval_seconds( + timestamp_batch: &[Timestamp], + chunk_start: usize, + offset: usize, +) -> u32 { + if chunk_start + offset == 0 { + return 0; + } + + let current_index = offset + usize::from(chunk_start > 0); + (*timestamp_batch[current_index]).saturating_sub(*timestamp_batch[current_index - 1]) +} + +#[inline(always)] +fn coindays_created(supply: Sats, interval_seconds: u32) -> StoredF64 { + StoredF64::from(f64::from(Bitcoin::from(supply)) * interval_seconds as f64 / ONE_DAY_IN_SEC_F64) +} + +fn age_bounds_days() -> [(f64, f64); AGE_COHORT_COUNT] { + let mut bounds = AGE_RANGE_BOUNDS.iter(); + std::array::from_fn(|index| { + let bound = bounds.next().unwrap(); + let lower = bound.start as f64 / HOURS_PER_DAY; + let width = if index + 1 < AGE_COHORT_COUNT { + (bound.end - bound.start) as f64 / HOURS_PER_DAY + } else { + 0.0 + }; + (lower, width) + }) +} + +fn allocate_consumed_coindays( + transfer_volume_btc: [f64; AGE_COHORT_COUNT], + coindays_destroyed: [f64; AGE_COHORT_COUNT], + bounds: &[(f64, f64); AGE_COHORT_COUNT], +) -> [f64; AGE_COHORT_COUNT] { + let mut result = [0.0; AGE_COHORT_COUNT]; + let mut older_transfer_volume = 0.0; + + for index in (0..AGE_COHORT_COUNT).rev() { + let (lower_days, width_days) = bounds[index]; + let within_cohort = + (coindays_destroyed[index] - transfer_volume_btc[index] * lower_days).max(0.0); + result[index] = within_cohort + older_transfer_volume * width_days; + older_transfer_volume += transfer_volume_btc[index]; + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn created_coindays_use_the_exact_monotonic_block_interval() { + let created = f64::from(coindays_created(Sats::ONE_BTC, 6 * 60 * 60)); + + assert!((created - 0.25).abs() < 1e-12); + } + + #[test] + fn monotonic_interval_handles_initial_and_resumed_chunks() { + let initial = [ + Timestamp::from(100_u32), + Timestamp::from(160_u32), + Timestamp::from(220_u32), + ]; + let resumed = [Timestamp::from(160_u32), Timestamp::from(220_u32)]; + + assert_eq!(monotonic_interval_seconds(&initial, 0, 0), 0); + assert_eq!(monotonic_interval_seconds(&initial, 0, 1), 60); + assert_eq!(monotonic_interval_seconds(&resumed, 2, 0), 60); + } + + #[test] + fn destruction_at_a_boundary_stays_in_the_ranges_already_traversed() { + let mut volumes = [0.0; AGE_COHORT_COUNT]; + let mut cdd = [0.0; AGE_COHORT_COUNT]; + volumes[2] = 1.0; + cdd[2] = 1.0; + + let allocated = allocate_consumed_coindays(volumes, cdd, &age_bounds_days()); + + assert!((allocated[0] - 1.0 / HOURS_PER_DAY).abs() < 1e-12); + assert!((allocated[1] - 23.0 / HOURS_PER_DAY).abs() < 1e-12); + assert!(allocated[2].abs() < 1e-12); + assert!((allocated.iter().sum::() - 1.0).abs() < 1e-12); + } + + #[test] + fn consumed_coindays_cover_every_traversed_cohort() { + let mut volumes = [0.0; AGE_COHORT_COUNT]; + let mut cdd = [0.0; AGE_COHORT_COUNT]; + volumes[2] = 2.0; + cdd[2] = 20.0; + + let allocated = allocate_consumed_coindays(volumes, cdd, &age_bounds_days()); + + assert!((allocated[0] - 2.0 / HOURS_PER_DAY).abs() < 1e-12); + assert!((allocated[1] - 46.0 / HOURS_PER_DAY).abs() < 1e-12); + assert!((allocated[2] - 18.0).abs() < 1e-12); + assert!((allocated.iter().sum::() - 20.0).abs() < 1e-12); + } + + #[test] + fn allocated_coindays_conserve_mixed_cohort_destruction() { + let bounds = age_bounds_days(); + let mut volumes = [0.0; AGE_COHORT_COUNT]; + let mut cdd = [0.0; AGE_COHORT_COUNT]; + + for index in [0, 1, 2, 10, 20] { + let (lower, width) = bounds[index]; + let volume = index as f64 + 1.0; + let age = lower + if width > 0.0 { width / 2.0 } else { 30.0 }; + volumes[index] = volume; + cdd[index] = volume * age; + } + + let allocated = allocate_consumed_coindays(volumes, cdd, &bounds); + + assert!((allocated.iter().sum::() - cdd.iter().sum::()).abs() < 1e-9); + } +} diff --git a/crates/brk_computer/src/frameworks/cointime/age_range/import.rs b/crates/brk_computer/src/frameworks/cointime/age_range/import.rs new file mode 100644 index 000000000..01d5f865f --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/age_range/import.rs @@ -0,0 +1,66 @@ +use brk_cohort::{AgeRange, CohortContext}; +use brk_error::Result; +use brk_types::Version; +use vecdb::Database; + +use super::{CohortVecs, Vecs}; +use crate::{ + indexes, + internal::{PerBlockCumulativeRolling, WindowStartVec, Windows}, +}; + +use super::super::{SupplyBaseVecs, activity::DerivedVecs as ActivityDerivedVecs}; + +const VERSION: Version = Version::ONE; + +impl CohortVecs { + fn forced_import( + db: &Database, + name: &str, + version: Version, + indexes: &indexes::Vecs, + cached_starts: &Windows<&WindowStartVec>, + ) -> Result { + Ok(Self { + coindays_created: PerBlockCumulativeRolling::forced_import( + db, + &format!("{name}_coindays_created"), + version, + indexes, + cached_starts, + )?, + coindays_consumed: PerBlockCumulativeRolling::forced_import( + db, + &format!("{name}_coindays_consumed"), + version, + indexes, + cached_starts, + )?, + coindays_stored: PerBlockCumulativeRolling::forced_import( + db, + &format!("{name}_coindays_stored"), + version, + indexes, + cached_starts, + )?, + activity: ActivityDerivedVecs::forced_import_with_prefix(db, name, version, indexes)?, + supply: SupplyBaseVecs::forced_import_with_prefix(db, name, version, indexes)?, + }) + } +} + +impl Vecs { + pub(crate) fn forced_import( + db: &Database, + parent_version: Version, + indexes: &indexes::Vecs, + cached_starts: &Windows<&WindowStartVec>, + ) -> Result { + let version = parent_version + VERSION; + + Ok(Self(AgeRange::try_new(|_, name| { + let name = CohortContext::Utxo.prefixed(name); + CohortVecs::forced_import(db, &name, version, indexes, cached_starts) + })?)) + } +} diff --git a/crates/brk_computer/src/cointime/value/mod.rs b/crates/brk_computer/src/frameworks/cointime/age_range/mod.rs similarity index 51% rename from crates/brk_computer/src/cointime/value/mod.rs rename to crates/brk_computer/src/frameworks/cointime/age_range/mod.rs index 1136f9ebd..f61257f67 100644 --- a/crates/brk_computer/src/cointime/value/mod.rs +++ b/crates/brk_computer/src/frameworks/cointime/age_range/mod.rs @@ -2,4 +2,4 @@ mod compute; mod import; mod vecs; -pub use vecs::Vecs; +pub use vecs::{CohortVecs, Vecs}; diff --git a/crates/brk_computer/src/frameworks/cointime/age_range/vecs.rs b/crates/brk_computer/src/frameworks/cointime/age_range/vecs.rs new file mode 100644 index 000000000..a9787b72d --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/age_range/vecs.rs @@ -0,0 +1,25 @@ +use brk_cohort::AgeRange; +use brk_traversable::Traversable; +use brk_types::StoredF64; +use derive_more::{Deref, DerefMut}; +use vecdb::{Rw, StorageMode}; + +use crate::internal::PerBlockCumulativeRolling; + +use super::super::{SupplyBaseVecs, activity::DerivedVecs as ActivityDerivedVecs}; + +#[derive(Deref, DerefMut, Traversable)] +pub struct CohortVecs { + pub coindays_created: PerBlockCumulativeRolling, + pub coindays_consumed: PerBlockCumulativeRolling, + pub coindays_stored: PerBlockCumulativeRolling, + #[deref] + #[deref_mut] + #[traversable(flatten)] + pub activity: ActivityDerivedVecs, + pub supply: SupplyBaseVecs, +} + +#[derive(Deref, DerefMut, Traversable)] +#[traversable(transparent)] +pub struct Vecs(pub AgeRange>); diff --git a/crates/brk_computer/src/cointime/cap/compute.rs b/crates/brk_computer/src/frameworks/cointime/cap/compute.rs similarity index 100% rename from crates/brk_computer/src/cointime/cap/compute.rs rename to crates/brk_computer/src/frameworks/cointime/cap/compute.rs diff --git a/crates/brk_computer/src/cointime/cap/import.rs b/crates/brk_computer/src/frameworks/cointime/cap/import.rs similarity index 100% rename from crates/brk_computer/src/cointime/cap/import.rs rename to crates/brk_computer/src/frameworks/cointime/cap/import.rs diff --git a/crates/brk_computer/src/cointime/adjusted/mod.rs b/crates/brk_computer/src/frameworks/cointime/cap/mod.rs similarity index 100% rename from crates/brk_computer/src/cointime/adjusted/mod.rs rename to crates/brk_computer/src/frameworks/cointime/cap/mod.rs diff --git a/crates/brk_computer/src/cointime/cap/vecs.rs b/crates/brk_computer/src/frameworks/cointime/cap/vecs.rs similarity index 100% rename from crates/brk_computer/src/cointime/cap/vecs.rs rename to crates/brk_computer/src/frameworks/cointime/cap/vecs.rs diff --git a/crates/brk_computer/src/cointime/compute.rs b/crates/brk_computer/src/frameworks/cointime/compute.rs similarity index 80% rename from crates/brk_computer/src/cointime/compute.rs rename to crates/brk_computer/src/frameworks/cointime/compute.rs index 1a81731d1..6d8fbf0ae 100644 --- a/crates/brk_computer/src/cointime/compute.rs +++ b/crates/brk_computer/src/frameworks/cointime/compute.rs @@ -3,13 +3,14 @@ use brk_indexer::Indexer; use vecdb::Exit; use super::Vecs; -use crate::{blocks, distribution, mining, price, supply}; +use crate::{blocks, distribution, indexes, mining, price, supply}; impl Vecs { #[allow(clippy::too_many_arguments)] pub(crate) fn compute( &mut self, indexer: &Indexer, + indexes: &indexes::Vecs, prices: &price::Vecs, blocks: &blocks::Vecs, mining: &mining::Vecs, @@ -21,12 +22,20 @@ impl Vecs { // Activity computes first (liveliness, vaultedness, etc.) self.activity.compute(indexer, distribution, exit)?; + self.age_range + .compute(indexer, indexes, prices, distribution, exit)?; - // Phase 2: supply, adjusted, value are independent (all depend only on activity) + // Phase 2: supply, adjusted, and value are independent. let (r1, r2) = rayon::join( || { - self.supply - .compute(indexer, prices, distribution, &self.activity, exit) + self.supply.compute( + indexer, + prices, + distribution, + &self.activity, + &self.age_range, + exit, + ) }, || { rayon::join( diff --git a/crates/brk_computer/src/cointime/import.rs b/crates/brk_computer/src/frameworks/cointime/import.rs similarity index 85% rename from crates/brk_computer/src/cointime/import.rs rename to crates/brk_computer/src/frameworks/cointime/import.rs index 8b16b7eac..8677c8f36 100644 --- a/crates/brk_computer/src/cointime/import.rs +++ b/crates/brk_computer/src/frameworks/cointime/import.rs @@ -9,8 +9,8 @@ use crate::{ }; use super::{ - ActivityVecs, AdjustedVecs, CapVecs, DB_NAME, PricesVecs, ReserveRiskVecs, SupplyVecs, - ValueVecs, Vecs, + ActivityVecs, AdjustedVecs, AgeRangeVecs, CapVecs, DB_NAME, PricesVecs, ReserveRiskVecs, + SupplyVecs, ValueVecs, Vecs, }; use crate::internal::{WindowStartVec, Windows}; @@ -26,6 +26,7 @@ impl Vecs { let version = parent_version; let v1 = version + Version::ONE; let activity = ActivityVecs::forced_import(&db, version, indexes, cached_starts)?; + let age_range = AgeRangeVecs::forced_import(&db, version, indexes, cached_starts)?; let supply = SupplyVecs::forced_import(&db, v1, indexes)?; let value = ValueVecs::forced_import(&db, v1, indexes, cached_starts)?; let cap = CapVecs::forced_import(&db, version + Version::TWO, indexes)?; @@ -36,6 +37,7 @@ impl Vecs { let this = Self { db, activity, + age_range, supply, value, cap, diff --git a/crates/brk_computer/src/cointime/mod.rs b/crates/brk_computer/src/frameworks/cointime/mod.rs similarity index 83% rename from crates/brk_computer/src/cointime/mod.rs rename to crates/brk_computer/src/frameworks/cointime/mod.rs index 19b347a02..a2da534a8 100644 --- a/crates/brk_computer/src/cointime/mod.rs +++ b/crates/brk_computer/src/frameworks/cointime/mod.rs @@ -1,5 +1,6 @@ pub mod activity; pub mod adjusted; +pub mod age_range; pub mod cap; pub mod prices; pub mod reserve_risk; @@ -14,10 +15,11 @@ use vecdb::{Database, Rw, StorageMode}; pub use activity::Vecs as ActivityVecs; pub use adjusted::Vecs as AdjustedVecs; +pub use age_range::Vecs as AgeRangeVecs; pub use cap::Vecs as CapVecs; pub use prices::Vecs as PricesVecs; pub use reserve_risk::Vecs as ReserveRiskVecs; -pub use supply::Vecs as SupplyVecs; +pub use supply::{BaseVecs as SupplyBaseVecs, Vecs as SupplyVecs}; pub use value::Vecs as ValueVecs; pub const DB_NAME: &str = "cointime"; @@ -28,6 +30,7 @@ pub struct Vecs { pub(crate) db: Database, pub activity: ActivityVecs, + pub age_range: AgeRangeVecs, pub supply: SupplyVecs, pub value: ValueVecs, pub cap: CapVecs, diff --git a/crates/brk_computer/src/cointime/prices/compute.rs b/crates/brk_computer/src/frameworks/cointime/prices/compute.rs similarity index 100% rename from crates/brk_computer/src/cointime/prices/compute.rs rename to crates/brk_computer/src/frameworks/cointime/prices/compute.rs diff --git a/crates/brk_computer/src/cointime/prices/import.rs b/crates/brk_computer/src/frameworks/cointime/prices/import.rs similarity index 100% rename from crates/brk_computer/src/cointime/prices/import.rs rename to crates/brk_computer/src/frameworks/cointime/prices/import.rs diff --git a/crates/brk_computer/src/cointime/cap/mod.rs b/crates/brk_computer/src/frameworks/cointime/prices/mod.rs similarity index 100% rename from crates/brk_computer/src/cointime/cap/mod.rs rename to crates/brk_computer/src/frameworks/cointime/prices/mod.rs diff --git a/crates/brk_computer/src/cointime/prices/vecs.rs b/crates/brk_computer/src/frameworks/cointime/prices/vecs.rs similarity index 100% rename from crates/brk_computer/src/cointime/prices/vecs.rs rename to crates/brk_computer/src/frameworks/cointime/prices/vecs.rs diff --git a/crates/brk_computer/src/cointime/reserve_risk/compute.rs b/crates/brk_computer/src/frameworks/cointime/reserve_risk/compute.rs similarity index 100% rename from crates/brk_computer/src/cointime/reserve_risk/compute.rs rename to crates/brk_computer/src/frameworks/cointime/reserve_risk/compute.rs diff --git a/crates/brk_computer/src/cointime/reserve_risk/import.rs b/crates/brk_computer/src/frameworks/cointime/reserve_risk/import.rs similarity index 100% rename from crates/brk_computer/src/cointime/reserve_risk/import.rs rename to crates/brk_computer/src/frameworks/cointime/reserve_risk/import.rs diff --git a/crates/brk_computer/src/cointime/prices/mod.rs b/crates/brk_computer/src/frameworks/cointime/reserve_risk/mod.rs similarity index 100% rename from crates/brk_computer/src/cointime/prices/mod.rs rename to crates/brk_computer/src/frameworks/cointime/reserve_risk/mod.rs diff --git a/crates/brk_computer/src/cointime/reserve_risk/vecs.rs b/crates/brk_computer/src/frameworks/cointime/reserve_risk/vecs.rs similarity index 100% rename from crates/brk_computer/src/cointime/reserve_risk/vecs.rs rename to crates/brk_computer/src/frameworks/cointime/reserve_risk/vecs.rs diff --git a/crates/brk_computer/src/frameworks/cointime/supply/compute.rs b/crates/brk_computer/src/frameworks/cointime/supply/compute.rs new file mode 100644 index 000000000..41db70e60 --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/supply/compute.rs @@ -0,0 +1,168 @@ +use brk_error::Result; +use brk_indexer::Indexer; +use brk_types::{Height, Sats, StoredF64, Version}; +use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, WritableVec}; + +use super::super::{activity, age_range}; +use super::{BaseVecs, Vecs}; +use crate::{distribution, frameworks::WeightedRatio, price}; + +const WRITE_INTERVAL: usize = 10_000; + +impl BaseVecs { + #[allow(clippy::too_many_arguments)] + pub(crate) fn compute_from( + &mut self, + starting_height: Height, + prices: &price::Vecs, + total_supply: &impl ReadableVec, + liveliness: &impl ReadableVec, + vaultedness: &impl ReadableVec, + exit: &Exit, + ) -> Result<()> { + self.vaulted.sats.height.compute_multiply( + starting_height, + total_supply, + vaultedness, + exit, + )?; + + self.active.sats.height.compute_multiply( + starting_height, + total_supply, + liveliness, + exit, + )?; + + self.vaulted.compute(prices, starting_height, exit)?; + self.active.compute(prices, starting_height, exit)?; + + Ok(()) + } +} + +impl Vecs { + pub(crate) fn compute( + &mut self, + indexer: &Indexer, + prices: &price::Vecs, + distribution: &distribution::Vecs, + activity: &activity::Vecs, + age_range: &age_range::Vecs, + exit: &Exit, + ) -> Result<()> { + let starting_height = indexer.safe_lengths().height; + let circulating_supply = &distribution + .utxo_cohorts + .all + .metrics + .supply + .total + .sats + .height; + + self.base.compute_from( + starting_height, + prices, + circulating_supply, + &activity.liveliness.height, + &activity.vaultedness.height, + exit, + )?; + + let source_cohorts: Vec<_> = distribution.utxo_cohorts.age_range.iter().collect(); + let total_supplies: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.supply.total.sats.height) + .collect(); + let loss_supplies: Vec<_> = source_cohorts + .iter() + .map(|cohort| &cohort.metrics.supply.in_loss.sats.height) + .collect(); + let weights: Vec<_> = age_range + .iter() + .map(|cohort| &cohort.liveliness.height) + .collect(); + + self.compute_active_supply_in_loss_share( + starting_height, + &total_supplies, + &loss_supplies, + &weights, + exit, + ) + } + + fn compute_active_supply_in_loss_share( + &mut self, + starting_height: Height, + total_supplies: &[&S], + loss_supplies: &[&S], + weights: &[&W], + exit: &Exit, + ) -> Result<()> + where + S: ReadableVec, + W: ReadableVec, + { + debug_assert_eq!(total_supplies.len(), loss_supplies.len()); + debug_assert_eq!(total_supplies.len(), weights.len()); + + let source_version: Version = total_supplies + .iter() + .map(|vec| vec.version()) + .chain(loss_supplies.iter().map(|vec| vec.version())) + .chain(weights.iter().map(|vec| vec.version())) + .sum(); + let output = &mut self.active_supply_in_loss_share.height; + output.validate_computed_version_or_reset(source_version)?; + + let start = output.len().min(usize::from(starting_height)); + output.truncate_if_needed_at(start)?; + let source_end = total_supplies + .iter() + .map(|vec| vec.len()) + .chain(loss_supplies.iter().map(|vec| vec.len())) + .chain(weights.iter().map(|vec| vec.len())) + .min() + .unwrap_or_default(); + + let mut chunk_start = start; + while chunk_start < source_end { + let chunk_end = (chunk_start + WRITE_INTERVAL).min(source_end); + let total_batches: Vec<_> = total_supplies + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + let loss_batches: Vec<_> = loss_supplies + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + let weight_batches: Vec<_> = weights + .iter() + .map(|vec| vec.collect_range_at(chunk_start, chunk_end)) + .collect(); + + for offset in 0..(chunk_end - chunk_start) { + let mut supply_in_loss = WeightedRatio::default(); + for cohort in 0..weights.len() { + let weight = f64::from(weight_batches[cohort][offset]); + supply_in_loss.add( + loss_batches[cohort][offset].as_u128() as f64, + total_batches[cohort][offset].as_u128() as f64, + weight, + ); + } + output.push(supply_in_loss.value()); + } + + { + let _lock = exit.lock(); + output.write()?; + } + chunk_start = chunk_end; + } + + Ok(()) + } +} diff --git a/crates/brk_computer/src/frameworks/cointime/supply/import.rs b/crates/brk_computer/src/frameworks/cointime/supply/import.rs new file mode 100644 index 000000000..a52afb481 --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/supply/import.rs @@ -0,0 +1,49 @@ +use brk_error::Result; +use brk_types::Version; +use vecdb::Database; + +use super::{BaseVecs, Vecs}; +use crate::{ + indexes, + internal::{PerBlock, ValuePerBlock}, +}; + +impl BaseVecs { + pub(crate) fn forced_import_with_prefix( + db: &Database, + prefix: &str, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let name = |metric: &str| { + if prefix.is_empty() { + metric.to_owned() + } else { + format!("{prefix}_{metric}") + } + }; + + Ok(Self { + vaulted: ValuePerBlock::forced_import(db, &name("vaulted_supply"), version, indexes)?, + active: ValuePerBlock::forced_import(db, &name("active_supply"), version, indexes)?, + }) + } +} + +impl Vecs { + pub(crate) fn forced_import( + db: &Database, + version: Version, + indexes: &indexes::Vecs, + ) -> Result { + Ok(Self { + base: BaseVecs::forced_import_with_prefix(db, "", version, indexes)?, + active_supply_in_loss_share: PerBlock::forced_import( + db, + "cointime_supply_in_loss_share", + version, + indexes, + )?, + }) + } +} diff --git a/crates/brk_computer/src/cointime/supply/mod.rs b/crates/brk_computer/src/frameworks/cointime/supply/mod.rs similarity index 52% rename from crates/brk_computer/src/cointime/supply/mod.rs rename to crates/brk_computer/src/frameworks/cointime/supply/mod.rs index 1136f9ebd..b1e6a2b21 100644 --- a/crates/brk_computer/src/cointime/supply/mod.rs +++ b/crates/brk_computer/src/frameworks/cointime/supply/mod.rs @@ -2,4 +2,4 @@ mod compute; mod import; mod vecs; -pub use vecs::Vecs; +pub use vecs::{BaseVecs, Vecs}; diff --git a/crates/brk_computer/src/frameworks/cointime/supply/vecs.rs b/crates/brk_computer/src/frameworks/cointime/supply/vecs.rs new file mode 100644 index 000000000..feaf64644 --- /dev/null +++ b/crates/brk_computer/src/frameworks/cointime/supply/vecs.rs @@ -0,0 +1,22 @@ +use brk_traversable::Traversable; +use brk_types::StoredF64; +use derive_more::{Deref, DerefMut}; +use vecdb::{Rw, StorageMode}; + +use crate::internal::{PerBlock, ValuePerBlock}; + +#[derive(Traversable)] +pub struct BaseVecs { + pub vaulted: ValuePerBlock, + pub active: ValuePerBlock, +} + +#[derive(Deref, DerefMut, Traversable)] +pub struct Vecs { + #[deref] + #[deref_mut] + #[traversable(flatten)] + pub base: BaseVecs, + #[traversable(wrap = "active/in_loss", rename = "share")] + pub active_supply_in_loss_share: PerBlock, +} diff --git a/crates/brk_computer/src/cointime/value/compute.rs b/crates/brk_computer/src/frameworks/cointime/value/compute.rs similarity index 100% rename from crates/brk_computer/src/cointime/value/compute.rs rename to crates/brk_computer/src/frameworks/cointime/value/compute.rs diff --git a/crates/brk_computer/src/cointime/value/import.rs b/crates/brk_computer/src/frameworks/cointime/value/import.rs similarity index 100% rename from crates/brk_computer/src/cointime/value/import.rs rename to crates/brk_computer/src/frameworks/cointime/value/import.rs diff --git a/crates/brk_computer/src/cointime/reserve_risk/mod.rs b/crates/brk_computer/src/frameworks/cointime/value/mod.rs similarity index 100% rename from crates/brk_computer/src/cointime/reserve_risk/mod.rs rename to crates/brk_computer/src/frameworks/cointime/value/mod.rs diff --git a/crates/brk_computer/src/cointime/value/vecs.rs b/crates/brk_computer/src/frameworks/cointime/value/vecs.rs similarity index 100% rename from crates/brk_computer/src/cointime/value/vecs.rs rename to crates/brk_computer/src/frameworks/cointime/value/vecs.rs diff --git a/crates/brk_computer/src/frameworks/mod.rs b/crates/brk_computer/src/frameworks/mod.rs new file mode 100644 index 000000000..dbad3bab5 --- /dev/null +++ b/crates/brk_computer/src/frameworks/mod.rs @@ -0,0 +1,29 @@ +use brk_types::StoredF64; + +pub mod coinflow; +pub mod cointime; + +#[derive(Clone, Copy, Default)] +pub(crate) struct WeightedRatio { + numerator: f64, + denominator: f64, +} + +impl WeightedRatio { + #[inline] + pub(crate) fn add(&mut self, numerator: f64, denominator: f64, weight: f64) { + if weight.is_finite() && weight > 0.0 { + self.numerator += numerator * weight; + self.denominator += denominator * weight; + } + } + + #[inline] + pub(crate) fn value(&self) -> StoredF64 { + if self.denominator > 0.0 { + StoredF64::from((self.numerator / self.denominator).clamp(0.0, 1.0)) + } else { + StoredF64::NAN + } + } +} diff --git a/crates/brk_computer/src/internal/db_utils.rs b/crates/brk_computer/src/internal/db_utils.rs index 003c63983..42f5b7f54 100644 --- a/crates/brk_computer/src/internal/db_utils.rs +++ b/crates/brk_computer/src/internal/db_utils.rs @@ -2,7 +2,8 @@ use std::path::Path; use brk_error::Result; use brk_traversable::Traversable; -use vecdb::{Database, PAGE_SIZE}; +use brk_types::Version; +use vecdb::{AnyStoredVec, Database, PAGE_SIZE}; pub(crate) fn open_db( parent_path: &Path, @@ -24,3 +25,17 @@ pub(crate) fn finalize_db(db: &Database, traversable: &impl Traversable) -> Resu db.compact()?; Ok(()) } + +pub(crate) fn validate_any_computed_version_or_reset( + vec: &mut dyn AnyStoredVec, + dependency_version: Version, +) -> Result<()> { + let computed_version = vec.header().vec_version() + dependency_version; + if computed_version != vec.header().computed_version() { + vec.mut_header().update_computed_version(computed_version); + if !vec.is_empty() { + vec.any_reset()?; + } + } + Ok(()) +} diff --git a/crates/brk_computer/src/internal/per_block/percent/cumulative_rolling.rs b/crates/brk_computer/src/internal/per_block/percent/cumulative_rolling.rs index 7d72277a6..0ea678419 100644 --- a/crates/brk_computer/src/internal/per_block/percent/cumulative_rolling.rs +++ b/crates/brk_computer/src/internal/per_block/percent/cumulative_rolling.rs @@ -85,15 +85,7 @@ impl PercentCumulativeRolling { starting_height: Height, exit: &Exit, ) -> Result<()> { - self.compute_binary::< - StoredU64, - StoredU64, - RatioU64, - _, - _, - _, - _, - >( + self.compute_binary::, _, _, _, _>( starting_height, &numerator.cumulative.height, &denominator.cumulative.height, diff --git a/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs b/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs index 09486c230..78f777f59 100644 --- a/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs +++ b/crates/brk_computer/src/internal/per_block/ratio/percentiles.rs @@ -159,11 +159,7 @@ impl RatioPerBlockPercentiles { self.$band .price .cents - .compute_binary::< - Cents, - PartsPerMillion32, - PriceTimesRatio, - >( + .compute_binary::>( starting_lengths.height, series_price, &self.$band.ratio.raw.height, diff --git a/crates/brk_computer/src/internal/transform/mod.rs b/crates/brk_computer/src/internal/transform/mod.rs index fb3cad604..07512ba5b 100644 --- a/crates/brk_computer/src/internal/transform/mod.rs +++ b/crates/brk_computer/src/internal/transform/mod.rs @@ -16,13 +16,13 @@ pub use currency::{ NegCentsUnsignedToDollars, SatsSignedToBitcoin, SatsToBitcoin, SatsToCents, }; pub use derived::{ - Days1, Days7, Days30, Days365, DaysToYears, PriceTimesRatio, PriceTimesRatioCents, RatioCents64, - TimesSqrt, + Days1, Days7, Days30, Days365, DaysToYears, PriceTimesRatio, PriceTimesRatioCents, + RatioCents64, TimesSqrt, }; pub use fixed_ratio::{FixedToPercent, FixedToRatio}; pub use ratio::{ - RatioCents, RatioCentsSignedCents, RatioCentsSignedDollars, RatioDiffCents, - RatioDiffDollars, RatioDiffF32, RatioDollars, RatioSats, RatioU64, RatioU64F32, + RatioCents, RatioCentsSignedCents, RatioCentsSignedDollars, RatioDiffCents, RatioDiffDollars, + RatioDiffF32, RatioDollars, RatioSats, RatioU64, RatioU64F32, }; pub use specialized::{ BlockCountTarget1m, BlockCountTarget1w, BlockCountTarget1y, BlockCountTarget24h, diff --git a/crates/brk_computer/src/lib.rs b/crates/brk_computer/src/lib.rs index 44146b707..4ca0014d1 100644 --- a/crates/brk_computer/src/lib.rs +++ b/crates/brk_computer/src/lib.rs @@ -10,9 +10,9 @@ use tracing::info; use vecdb::{AnyExportableVec, Exit, Ro, Rw, StorageMode}; mod blocks; -mod cointime; mod constants; mod distribution; +mod frameworks; pub mod indexes; mod indicators; mod inputs; @@ -20,6 +20,7 @@ mod internal; mod investing; mod market; mod mining; +mod models; mod op_return; mod outputs; mod pools; @@ -27,12 +28,17 @@ pub mod price; mod supply; mod transactions; +use frameworks::{coinflow, cointime}; +use models::bedrock; + #[derive(Traversable)] pub struct Computer { pub blocks: Box>, pub mining: Box>, pub transactions: Box>, pub cointime: Box>, + pub coinflow: Box>, + pub bedrock: Box>, pub constants: Box, pub indexes: Box>, pub indicators: Box>, @@ -89,8 +95,8 @@ impl Computer { let cached_starts = blocks.lookback.cached_window_starts(); - let (inputs, outputs, mining, transactions, pools, cointime, op_return) = timed( - "Imported inputs/outputs/mining/tx/pools/cointime/op_return", + let (inputs, outputs, mining, transactions, pools, cointime, coinflow, op_return) = timed( + "Imported inputs/outputs/mining/tx/pools/cointime/coinflow/op_return", || { thread::scope(|s| -> Result<_> { let inputs_handle = big_thread().spawn_scoped(s, || -> Result<_> { @@ -146,6 +152,12 @@ impl Computer { &cached_starts, )?); + let coinflow = Box::new(coinflow::Vecs::forced_import( + &computed_path, + VERSION, + &indexes, + )?); + let op_return_handle = big_thread().spawn_scoped(s, || -> Result<_> { Ok(Box::new(op_return::Vecs::forced_import( &computed_path, @@ -169,6 +181,7 @@ impl Computer { transactions, pools, cointime, + coinflow, op_return, )) }) @@ -229,6 +242,14 @@ impl Computer { )?)) })?; + let bedrock = timed("Imported bedrock", || -> Result<_> { + Ok(Box::new(bedrock::Vecs::forced_import( + &computed_path, + VERSION, + &indexes, + )?)) + })?; + info!("Total import time: {:?}", import_start.elapsed()); let this = Self { @@ -243,6 +264,8 @@ impl Computer { supply, pools, cointime, + coinflow, + bedrock, indexes, inputs, price, @@ -262,6 +285,8 @@ impl Computer { mining::DB_NAME, transactions::DB_NAME, cointime::DB_NAME, + coinflow::DB_NAME, + bedrock::DB_NAME, indicators::DB_NAME, indexes::DB_NAME, investing::DB_NAME, @@ -422,8 +447,8 @@ impl Computer { Ok(()) })?; - // Indicators doesn't depend on supply or cointime — run it in the - // background alongside supply + cointime to save a scope barrier. + // Indicators doesn't depend on supply or either framework — run it in + // the background alongside their sequential computation. thread::scope(|scope| -> Result<()> { let indicators = scope.spawn(|| { timed("Computed indicators", || { @@ -454,6 +479,7 @@ impl Computer { timed("Computed cointime", || { self.cointime.compute( indexer, + &self.indexes, &self.price, &self.blocks, &self.mining, @@ -463,6 +489,28 @@ impl Computer { ) })?; + timed("Computed coinflow", || { + self.coinflow.compute( + indexer, + &self.indexes, + &self.price, + &self.distribution, + &self.cointime, + exit, + ) + })?; + + timed("Computed bedrock", || { + self.bedrock.compute( + indexer, + &self.indexes, + &self.distribution, + &self.cointime, + &self.coinflow, + exit, + ) + })?; + indicators.join().unwrap()?; Ok(()) })?; @@ -516,6 +564,8 @@ impl_iter_named!( mining, transactions, cointime, + coinflow, + bedrock, constants, indicators, indexes, diff --git a/crates/brk_computer/src/mining/rewards/compute.rs b/crates/brk_computer/src/mining/rewards/compute.rs index 84d0bfe61..b3ecb3749 100644 --- a/crates/brk_computer/src/mining/rewards/compute.rs +++ b/crates/brk_computer/src/mining/rewards/compute.rs @@ -1,8 +1,6 @@ use brk_error::Result; use brk_indexer::Indexer; -use brk_types::{ - CheckedSub, Dollars, Halving, PartsPerMillion32, PartsPerMillion64, Sats, -}; +use brk_types::{CheckedSub, Dollars, Halving, PartsPerMillion32, PartsPerMillion64, Sats}; use vecdb::{Exit, ReadableVec, VecIndex}; use super::Vecs; diff --git a/crates/brk_computer/src/models/bedrock/compute.rs b/crates/brk_computer/src/models/bedrock/compute.rs new file mode 100644 index 000000000..bbcc846a6 --- /dev/null +++ b/crates/brk_computer/src/models/bedrock/compute.rs @@ -0,0 +1,646 @@ +use std::{cmp::Ordering, collections::BTreeMap, fs, path::Path}; + +use brk_cohort::{AGE_RANGE_NAMES, CohortContext}; +use brk_error::Result; +use brk_indexer::Indexer; +use brk_types::{CentsCompact, Date, Day1, Dollars, StoredF64, UrpdRaw, Version}; +use vecdb::{AnyStoredVec, AnyVec, Exit, ReadableVec, VecValue, WritableVec}; + +use super::vecs::{Levels, MODE_COUNT, ModeVecs, Percentiles, Vecs}; +use crate::{ + distribution, + frameworks::{ + coinflow::{self, AGE_COHORT_COUNT, AgeBand, HORIZON_COUNT, HORIZON_DAYS, age_bounds_days}, + cointime, + }, + indexes, + internal::db_utils::validate_any_computed_version_or_reset, +}; + +const MIN_CALIBRATION_DAYS: usize = 365; +const WRITE_INTERVAL_DAYS: usize = 100; +const PERCENTILE_COUNT: usize = 5; +const LEVEL_COUNT: usize = 9; +const PERCENTILES: [f64; PERCENTILE_COUNT] = [0.95, 0.98, 0.99, 0.995, 0.999]; +const LEVEL_PERCENTILES: [f64; LEVEL_COUNT] = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]; + +const RAW_MODE: usize = 0; +const COINTIME_MODE: usize = 1; +const COINFLOW_MODE: usize = 2; +const COINFLOW_HORIZON_START: usize = 3; + +type Thresholds = [Option<[f64; PERCENTILE_COUNT]>; MODE_COUNT]; +type ModeWeights = [Option<[f64; AGE_COHORT_COUNT]>; MODE_COUNT]; + +struct DayResult { + loss_threshold: [[StoredF64; PERCENTILE_COUNT]; MODE_COUNT], + floor: [[Dollars; PERCENTILE_COUNT]; MODE_COUNT], + level: [[Dollars; LEVEL_COUNT]; MODE_COUNT], +} + +impl DayResult { + fn from_thresholds(thresholds: &Thresholds) -> Self { + Self { + loss_threshold: thresholds.map(|thresholds| { + thresholds + .map(|values| values.map(StoredF64::from)) + .unwrap_or([StoredF64::NAN; PERCENTILE_COUNT]) + }), + floor: [[Dollars::NAN; PERCENTILE_COUNT]; MODE_COUNT], + level: [[Dollars::NAN; LEVEL_COUNT]; MODE_COUNT], + } + } +} + +struct Calibration { + histories: [Vec; MODE_COUNT], +} + +impl Calibration { + fn from_sources( + raw: &impl ReadableVec>, + weighted: &[&impl ReadableVec>], + end: usize, + ) -> Self + where + T: VecValue, + U: VecValue, + f64: From + From, + { + let mut histories = std::array::from_fn(|_| Vec::new()); + histories[RAW_MODE] = collect_loss_history(raw, end); + for (history, source) in histories[1..].iter_mut().zip(weighted) { + *history = collect_loss_history(*source, end); + } + Self { histories } + } + + fn thresholds(&self, current: &[Option; MODE_COUNT]) -> Thresholds { + std::array::from_fn(|mode| { + (current[mode].is_some() && self.histories[mode].len() >= MIN_CALIBRATION_DAYS).then( + || { + PERCENTILES.map(|percentile| { + quantile(&self.histories[mode], percentile).expect("non-empty history") + }) + }, + ) + }) + } + + fn observe(&mut self, shares: [Option; MODE_COUNT]) { + for (history, share) in self.histories.iter_mut().zip(shares) { + if let Some(share) = share { + insert_sorted(history, share.clamp(0.0, 1.0)); + } + } + } +} + +impl Percentiles { + fn as_mut_array(&mut self) -> [&mut T; PERCENTILE_COUNT] { + [ + &mut self.pct95, + &mut self.pct98, + &mut self.pct99, + &mut self.pct99_5, + &mut self.pct99_9, + ] + } +} + +impl Levels { + fn as_mut_array(&mut self) -> [&mut T; LEVEL_COUNT] { + [ + &mut self.pct10, + &mut self.pct20, + &mut self.pct30, + &mut self.pct40, + &mut self.pct50, + &mut self.pct60, + &mut self.pct70, + &mut self.pct80, + &mut self.pct90, + ] + } +} + +impl ModeVecs { + fn stored_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> { + let mut vecs: Vec<&mut dyn AnyStoredVec> = + Vec::with_capacity(2 * PERCENTILE_COUNT + LEVEL_COUNT); + vecs.extend( + self.loss_threshold + .as_mut_array() + .into_iter() + .map(|vec| &mut vec.day1 as &mut dyn AnyStoredVec), + ); + vecs.extend( + self.floor + .as_mut_array() + .into_iter() + .map(|vec| &mut vec.day1 as &mut dyn AnyStoredVec), + ); + vecs.extend( + self.level + .as_mut_array() + .into_iter() + .map(|vec| &mut vec.day1 as &mut dyn AnyStoredVec), + ); + vecs + } + + fn push( + &mut self, + loss_threshold: [StoredF64; PERCENTILE_COUNT], + floor: [Dollars; PERCENTILE_COUNT], + level: [Dollars; LEVEL_COUNT], + ) { + for (vec, value) in self + .loss_threshold + .as_mut_array() + .into_iter() + .zip(loss_threshold) + { + vec.day1.push(value); + } + for (vec, value) in self.floor.as_mut_array().into_iter().zip(floor) { + vec.day1.push(value); + } + for (vec, value) in self.level.as_mut_array().into_iter().zip(level) { + vec.day1.push(value); + } + } +} + +impl Vecs { + pub(crate) fn compute( + &mut self, + indexer: &Indexer, + indexes: &indexes::Vecs, + distribution: &distribution::Vecs, + cointime: &cointime::Vecs, + coinflow: &coinflow::Vecs, + exit: &Exit, + ) -> Result<()> { + self.db.sync_bg_tasks()?; + + let cointime_liveliness: Vec<_> = cointime + .age_range + .iter() + .map(|cohort| &cohort.liveliness.day1) + .collect(); + let coinflow_mobility: Vec<_> = coinflow + .age_range + .iter() + .map(|cohort| &cohort.mobility.day1) + .collect(); + let coinflow_spending_rate: Vec<_> = coinflow + .age_range + .iter() + .map(|cohort| &cohort.spending_rate.day1) + .collect(); + let raw_loss_share = &distribution + .utxo_cohorts + .all + .metrics + .relative + .supply_in_loss_share + .raw + .day1; + let weighted_loss_shares: Vec<_> = [ + &cointime.supply.active_supply_in_loss_share.day1, + &coinflow.supply_in_loss_share.day1, + ] + .into_iter() + .chain( + coinflow + .horizon + .iter() + .map(|horizon| &horizon.supply_in_loss_share.day1), + ) + .collect(); + debug_assert_eq!(weighted_loss_shares.len(), MODE_COUNT - 1); + + let source_version: Version = std::iter::once(indexes.day1.date.version()) + .chain(std::iter::once(distribution.supply_state.version())) + .chain(std::iter::once(raw_loss_share.version())) + .chain(weighted_loss_shares.iter().map(|vec| vec.version())) + .chain(cointime_liveliness.iter().map(|vec| vec.version())) + .chain(coinflow_mobility.iter().map(|vec| vec.version())) + .chain(coinflow_spending_rate.iter().map(|vec| vec.version())) + .sum(); + + for vec in self.stored_vecs_mut() { + validate_any_computed_version_or_reset(vec, source_version)?; + } + + let source_end = std::iter::once(indexes.day1.date.len()) + .chain(std::iter::once(raw_loss_share.len())) + .chain(weighted_loss_shares.iter().map(|vec| vec.len())) + .chain(cointime_liveliness.iter().map(|vec| vec.len())) + .chain(coinflow_mobility.iter().map(|vec| vec.len())) + .chain(coinflow_spending_rate.iter().map(|vec| vec.len())) + .min() + .unwrap_or_default(); + let recompute_from = recompute_day(indexer, indexes) + .map(usize::from) + .unwrap_or_default(); + let start = self.minimum_len().min(recompute_from).min(source_end); + + for vec in self.stored_vecs_mut() { + vec.any_truncate_if_needed_at(start)?; + } + + let mut calibration = + Calibration::from_sources(raw_loss_share, &weighted_loss_shares, start); + let bounds = age_bounds_days(); + + for day_index in start..source_end { + let day = Day1::from(day_index); + let loss_shares = collect_loss_shares(raw_loss_share, &weighted_loss_shares, day); + let thresholds = calibration.thresholds(&loss_shares); + let mut result = DayResult::from_thresholds(&thresholds); + + if thresholds.iter().any(Option::is_some) + && let Some(date) = indexes.day1.date.collect_one(day) + { + let weights = mode_weights( + day, + &cointime_liveliness, + &coinflow_mobility, + &coinflow_spending_rate, + &bounds, + ); + if let Some(weighted) = + read_weighted_urpd(&distribution.states_path, date, &weights)? + { + evaluate_day(&weighted, &thresholds, &mut result); + } + } + calibration.observe(loss_shares); + + for (mode, output) in self.modes.as_mut_array().into_iter().enumerate() { + output.push( + result.loss_threshold[mode], + result.floor[mode], + result.level[mode], + ); + } + + if (day_index + 1).is_multiple_of(WRITE_INTERVAL_DAYS) || day_index + 1 == source_end { + let _lock = exit.lock(); + for vec in self.stored_vecs_mut() { + vec.write()?; + } + } + } + + let bedrock_exit = exit.clone(); + self.db.run_bg(move |db| { + let _lock = bedrock_exit.lock(); + db.compact_deferred_default() + }); + Ok(()) + } + + fn stored_vecs_mut(&mut self) -> Vec<&mut dyn AnyStoredVec> { + let mut vecs = Vec::with_capacity(MODE_COUNT * (2 * PERCENTILE_COUNT + LEVEL_COUNT)); + for mode in self.modes.as_mut_array() { + vecs.extend(mode.stored_vecs_mut()); + } + vecs + } + + fn minimum_len(&mut self) -> usize { + self.stored_vecs_mut() + .into_iter() + .map(|vec| vec.len()) + .min() + .unwrap_or_default() + } +} + +fn collect_loss_history(source: &impl ReadableVec>, end: usize) -> Vec +where + T: VecValue, + f64: From, +{ + let mut history: Vec<_> = source + .collect_range_at(0, end) + .into_iter() + .flatten() + .map(f64::from) + .filter(|value| value.is_finite()) + .collect(); + history.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal)); + history +} + +fn collect_loss_share(source: &impl ReadableVec>, day: Day1) -> Option +where + T: VecValue, + f64: From, +{ + source + .collect_one(day) + .flatten() + .map(f64::from) + .filter(|value| value.is_finite()) +} + +fn collect_loss_shares( + raw: &impl ReadableVec>, + weighted: &[&impl ReadableVec>], + day: Day1, +) -> [Option; MODE_COUNT] +where + T: VecValue, + U: VecValue, + f64: From + From, +{ + let mut shares = [None; MODE_COUNT]; + shares[RAW_MODE] = collect_loss_share(raw, day); + for (share, source) in shares[1..].iter_mut().zip(weighted) { + *share = collect_loss_share(*source, day); + } + shares +} + +fn recompute_day(indexer: &Indexer, indexes: &indexes::Vecs) -> Option { + let starting_height = indexer.safe_lengths().height; + indexes + .height + .day1 + .collect_one(starting_height) + .or_else(|| { + starting_height + .decremented() + .and_then(|height| indexes.height.day1.collect_one(height)) + }) +} + +fn mode_weights( + day: Day1, + cointime_liveliness: &[&impl ReadableVec>], + coinflow_mobility: &[&impl ReadableVec>], + coinflow_spending_rate: &[&impl ReadableVec>], + bounds: &[AgeBand; AGE_COHORT_COUNT], +) -> ModeWeights { + debug_assert_eq!(COINFLOW_HORIZON_START + HORIZON_COUNT, MODE_COUNT); + let mut weights = [None; MODE_COUNT]; + weights[RAW_MODE] = Some([1.0; AGE_COHORT_COUNT]); + weights[COINTIME_MODE] = collect_age_values(cointime_liveliness, day) + .map(|values| values.map(|v| v.clamp(0.0, 1.0))); + weights[COINFLOW_MODE] = + collect_age_values(coinflow_mobility, day).map(|values| values.map(|v| v.clamp(0.0, 1.0))); + + if let Some(hazards) = collect_age_values(coinflow_spending_rate, day) { + let hazards = hazards.map(|value| value.max(0.0)); + for (offset, horizon) in HORIZON_DAYS.iter().copied().enumerate() { + weights[COINFLOW_HORIZON_START + offset] = Some(std::array::from_fn(|age| { + coinflow::horizon_mobility(&hazards, age, horizon, bounds) + })); + } + } + weights +} + +fn collect_age_values( + sources: &[&impl ReadableVec>], + day: Day1, +) -> Option<[f64; AGE_COHORT_COUNT]> { + if sources.len() != AGE_COHORT_COUNT { + return None; + } + + let mut values = [0.0; AGE_COHORT_COUNT]; + for (value, source) in values.iter_mut().zip(sources) { + let collected = f64::from(source.collect_one(day).flatten()?); + if !collected.is_finite() { + return None; + } + *value = collected; + } + Some(values) +} + +fn read_weighted_urpd( + states_path: &Path, + date: Date, + weights: &ModeWeights, +) -> Result>> { + let mut weighted = BTreeMap::::new(); + + for (age, name) in AGE_RANGE_NAMES.iter().enumerate() { + let cohort = CohortContext::Utxo.prefixed(name.id); + let path = states_path.join(cohort).join("urpd").join(date.to_string()); + let bytes = fs::read(&path).map_err(|error| { + std::io::Error::new( + error.kind(), + format!("Cannot read URPD '{}': {error}", path.display()), + ) + })?; + let urpd = UrpdRaw::deserialize(&bytes)?; + + for (price, sats) in urpd.map { + let mass = u64::from(sats) as f64; + let bucket = weighted.entry(price).or_insert([0.0; MODE_COUNT]); + for (bucket, weights) in bucket.iter_mut().zip(weights) { + if let Some(weights) = weights { + *bucket += mass * weights[age]; + } + } + } + } + + Ok((!weighted.is_empty()).then_some(weighted)) +} + +fn evaluate_day( + weighted: &BTreeMap, + thresholds: &Thresholds, + result: &mut DayResult, +) { + let mut total_mass = [0.0; MODE_COUNT]; + let mut has_positive_cost = [false; MODE_COUNT]; + + for (price, buckets) in weighted { + for mode in 0..MODE_COUNT { + let mass = buckets[mode]; + total_mass[mode] += mass; + has_positive_cost[mode] |= price.inner() != 0 && mass > 0.0; + } + } + + for mode in 0..MODE_COUNT { + let denominator = total_mass[mode]; + let Some(thresholds) = thresholds[mode] else { + continue; + }; + if denominator <= 0.0 || !has_positive_cost[mode] { + continue; + } + + let mut remaining_loss = denominator; + let mut floors = [Dollars::NAN; PERCENTILE_COUNT]; + let mut p95_floor = None; + for (price, buckets) in weighted { + remaining_loss -= buckets[mode]; + let remaining_share = remaining_loss / denominator; + for percentile in 0..PERCENTILE_COUNT { + if floors[percentile].is_nan() && remaining_share <= thresholds[percentile] { + floors[percentile] = Dollars::from(*price); + if percentile == 0 { + p95_floor = Some(*price); + } + } + } + if floors.iter().all(|floor| !floor.is_nan()) { + break; + } + } + result.floor[mode] = floors; + if let Some(p95_floor) = p95_floor { + result.level[mode] = conditional_levels(weighted, mode, p95_floor); + } + } +} + +fn conditional_levels( + weighted: &BTreeMap, + mode: usize, + lower: CentsCompact, +) -> [Dollars; LEVEL_COUNT] { + let mut levels = [Dollars::NAN; LEVEL_COUNT]; + let total = weighted + .range(lower..) + .map(|(_, buckets)| buckets[mode]) + .filter(|mass| mass.is_finite() && *mass > 0.0) + .sum::(); + if !total.is_finite() || total <= 0.0 { + return levels; + } + + let mut cumulative = 0.0; + let mut percentile = 0; + for (price, buckets) in weighted.range(lower..) { + let mass = buckets[mode]; + if !mass.is_finite() || mass <= 0.0 { + continue; + } + cumulative += mass; + while percentile < LEVEL_COUNT && cumulative >= total * LEVEL_PERCENTILES[percentile] { + levels[percentile] = Dollars::from(*price); + percentile += 1; + } + if percentile == LEVEL_COUNT { + break; + } + } + levels +} + +fn quantile(sorted: &[f64], percentile: f64) -> Option { + if sorted.is_empty() { + return None; + } + let position = percentile.clamp(0.0, 1.0) * (sorted.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + let fraction = position - lower as f64; + Some(sorted[lower] * (1.0 - fraction) + sorted[upper] * fraction) +} + +fn insert_sorted(values: &mut Vec, value: f64) { + let index = values + .binary_search_by(|candidate| candidate.partial_cmp(&value).unwrap_or(Ordering::Less)) + .unwrap_or_else(|index| index); + values.insert(index, value); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quantile_linearly_interpolates() { + assert_eq!(quantile(&[0.0, 1.0], 0.95), Some(0.95)); + assert_eq!(quantile(&[], 0.95), None); + } + + #[test] + fn daily_loss_share_calibrates_the_floor() { + let weighted = BTreeMap::from([ + (CentsCompact::new(100), [50.0; MODE_COUNT]), + (CentsCompact::new(200), [50.0; MODE_COUNT]), + ]); + let mut calibration = Calibration { + histories: std::array::from_fn(|_| vec![0.5; MIN_CALIBRATION_DAYS]), + }; + let shares = [Some(0.5); MODE_COUNT]; + let thresholds = calibration.thresholds(&shares); + let mut result = DayResult::from_thresholds(&thresholds); + evaluate_day(&weighted, &thresholds, &mut result); + calibration.observe(shares); + + assert_eq!( + result.loss_threshold[COINFLOW_MODE], + [StoredF64::from(0.5); PERCENTILE_COUNT] + ); + assert_eq!( + result.floor[COINFLOW_MODE], + [Dollars::from(1.0); PERCENTILE_COUNT] + ); + assert_eq!( + result.level[COINFLOW_MODE], + [ + Dollars::from(1.0), + Dollars::from(1.0), + Dollars::from(1.0), + Dollars::from(1.0), + Dollars::from(1.0), + Dollars::from(2.0), + Dollars::from(2.0), + Dollars::from(2.0), + Dollars::from(2.0), + ] + ); + assert_eq!( + calibration.histories[COINFLOW_MODE].len(), + MIN_CALIBRATION_DAYS + 1 + ); + } + + #[test] + fn zero_cost_distribution_stays_missing() { + let weighted = BTreeMap::from([(CentsCompact::new(0), [100.0; MODE_COUNT])]); + let mut calibration = Calibration { + histories: std::array::from_fn(|_| vec![0.5; MIN_CALIBRATION_DAYS]), + }; + let shares = [Some(1.0); MODE_COUNT]; + let thresholds = calibration.thresholds(&shares); + let mut result = DayResult::from_thresholds(&thresholds); + evaluate_day(&weighted, &thresholds, &mut result); + calibration.observe(shares); + + assert_eq!(result.loss_threshold[RAW_MODE][0], StoredF64::from(0.5)); + assert!(result.floor[RAW_MODE][0].is_nan()); + assert_eq!( + calibration.histories[RAW_MODE].len(), + MIN_CALIBRATION_DAYS + 1 + ); + } + + #[test] + fn missing_framework_share_does_not_update_history() { + let mut calibration = Calibration { + histories: std::array::from_fn(|_| Vec::new()), + }; + let shares = [None; MODE_COUNT]; + let thresholds = calibration.thresholds(&shares); + calibration.observe(shares); + + assert_eq!(thresholds, [None; MODE_COUNT]); + assert!(calibration.histories[RAW_MODE].is_empty()); + } +} diff --git a/crates/brk_computer/src/models/bedrock/import.rs b/crates/brk_computer/src/models/bedrock/import.rs new file mode 100644 index 000000000..9f7c7f9c0 --- /dev/null +++ b/crates/brk_computer/src/models/bedrock/import.rs @@ -0,0 +1,105 @@ +use std::path::Path; + +use brk_error::Result; +use brk_types::{Dollars, StoredF64, Version}; +use vecdb::Database; + +use super::{ + DB_NAME, + urpd_metric::{UrpdMappings, UrpdMetric}, + vecs::{Levels, ModeVecs, Modes, Percentiles, Vecs}, +}; +use crate::{ + indexes, + internal::db_utils::{finalize_db, open_db}, +}; + +const VERSION: Version = Version::TWO; + +fn import_percentiles(mut import: impl FnMut(&str) -> Result) -> Result> { + Ok(Percentiles { + pct95: import("pct95")?, + pct98: import("pct98")?, + pct99: import("pct99")?, + pct99_5: import("pct99_5")?, + pct99_9: import("pct99_9")?, + }) +} + +fn import_levels(mut import: impl FnMut(&str) -> Result) -> Result> { + Ok(Levels { + pct10: import("pct10")?, + pct20: import("pct20")?, + pct30: import("pct30")?, + pct40: import("pct40")?, + pct50: import("pct50")?, + pct60: import("pct60")?, + pct70: import("pct70")?, + pct80: import("pct80")?, + pct90: import("pct90")?, + }) +} + +fn import_ratio( + db: &Database, + name: &str, + version: Version, + mappings: &UrpdMappings, +) -> Result> { + UrpdMetric::forced_import(db, name, version, mappings) +} + +fn import_price( + db: &Database, + name: &str, + version: Version, + mappings: &UrpdMappings, +) -> Result> { + UrpdMetric::forced_import(db, name, version, mappings) +} + +fn import_mode( + db: &Database, + name: &str, + version: Version, + mappings: &UrpdMappings, +) -> Result { + Ok(ModeVecs { + loss_threshold: import_percentiles(|percentile| { + import_ratio( + db, + &format!("{name}_loss_threshold_{percentile}"), + version, + mappings, + ) + })?, + floor: import_percentiles(|percentile| { + import_price(db, &format!("{name}_floor_{percentile}"), version, mappings) + })?, + level: import_levels(|percentile| { + import_price(db, &format!("{name}_level_{percentile}"), version, mappings) + })?, + }) +} + +impl Vecs { + pub(crate) fn forced_import( + parent_path: &Path, + parent_version: Version, + indexes: &indexes::Vecs, + ) -> Result { + let db = open_db(parent_path, DB_NAME, 50_000)?; + let version = parent_version + VERSION; + let mappings = UrpdMappings::new(indexes); + + let this = Self { + modes: Modes::try_from_fn(|name| { + import_mode(&db, &format!("bedrock_{name}"), version, &mappings) + })?, + db, + }; + + finalize_db(&this.db, &this)?; + Ok(this) + } +} diff --git a/crates/brk_computer/src/models/bedrock/mod.rs b/crates/brk_computer/src/models/bedrock/mod.rs new file mode 100644 index 000000000..4c4760c49 --- /dev/null +++ b/crates/brk_computer/src/models/bedrock/mod.rs @@ -0,0 +1,8 @@ +mod compute; +mod import; +mod urpd_metric; +mod vecs; + +pub use vecs::Vecs; + +pub const DB_NAME: &str = "bedrock"; diff --git a/crates/brk_computer/src/models/bedrock/urpd_metric.rs b/crates/brk_computer/src/models/bedrock/urpd_metric.rs new file mode 100644 index 000000000..14e21a85b --- /dev/null +++ b/crates/brk_computer/src/models/bedrock/urpd_metric.rs @@ -0,0 +1,570 @@ +use std::{marker::PhantomData, sync::Arc}; + +use brk_error::Result; +use brk_traversable::{Index, SeriesLeaf, SeriesLeafWithSchema, Traversable, TreeNode}; +use brk_types::{ + Date, Day1, Day3, Epoch, Halving, Height, Hour1, Hour4, Hour12, Minute10, Minute30, Month1, + Month3, Month6, Timestamp, Version, Week1, Year1, Year10, +}; +use schemars::JsonSchema; +use vecdb::{ + AnyExportableVec, AnyVec, Database, EagerVec, ImportableVec, LazyVecFrom1, PcoVec, + ReadableBoxedVec, ReadableCloneableVec, ReadableVec, Rw, StorageMode, TypedVec, VecIndex, + VecValue, short_type_name, +}; + +use crate::{indexes, internal::NumericValue}; + +type StoredDay = ::Stored>>; +type DayMapping = LazyVecFrom1; +type Repeated = UrpdView; +type Last = UrpdView; + +pub struct RepeatDay; +pub struct LastDay; + +#[derive(Clone)] +pub(crate) struct UrpdMappings { + height: DayMapping, + minute10: DayMapping, + minute30: DayMapping, + hour1: DayMapping, + hour4: DayMapping, + hour12: DayMapping, + day3: DayMapping, + week1: DayMapping, + month1: DayMapping, + month3: DayMapping, + month6: DayMapping, + year1: DayMapping, + year10: DayMapping, + halving: DayMapping, + epoch: DayMapping, +} + +impl UrpdMappings { + pub(crate) fn new(indexes: &indexes::Vecs) -> Self { + let height = LazyVecFrom1::init( + "day1", + Version::ZERO, + indexes.height.day1.read_only_boxed_clone(), + |_, day| day, + ); + + Self { + height, + minute10: timestamp_mapping(indexes.timestamp.minute10.read_only_boxed_clone()), + minute30: timestamp_mapping(indexes.timestamp.minute30.read_only_boxed_clone()), + hour1: timestamp_mapping(indexes.timestamp.hour1.read_only_boxed_clone()), + hour4: timestamp_mapping(indexes.timestamp.hour4.read_only_boxed_clone()), + hour12: timestamp_mapping(indexes.timestamp.hour12.read_only_boxed_clone()), + day3: date_mapping(indexes.day3.date.read_only_boxed_clone()), + week1: date_mapping(indexes.week1.date.read_only_boxed_clone()), + month1: date_mapping(indexes.month1.date.read_only_boxed_clone()), + month3: date_mapping(indexes.month3.date.read_only_boxed_clone()), + month6: date_mapping(indexes.month6.date.read_only_boxed_clone()), + year1: date_mapping(indexes.year1.date.read_only_boxed_clone()), + year10: date_mapping(indexes.year10.date.read_only_boxed_clone()), + halving: timestamp_mapping(indexes.timestamp.halving.read_only_boxed_clone()), + epoch: timestamp_mapping(indexes.timestamp.epoch.read_only_boxed_clone()), + } + } +} + +fn timestamp_mapping( + source: ReadableBoxedVec, +) -> DayMapping { + LazyVecFrom1::init("day1", Version::ZERO, source, |_, timestamp| { + Day1::try_from(Date::from(timestamp)).unwrap_or_default() + }) +} + +fn date_mapping(source: ReadableBoxedVec) -> DayMapping { + LazyVecFrom1::init("day1", Version::ZERO, source, |_, date| { + Day1::try_from(date).unwrap_or_default() + }) +} + +#[derive(Clone, Traversable)] +#[traversable(merge)] +pub struct UrpdViews +where + T: NumericValue + JsonSchema, +{ + pub height: Repeated, + pub minute10: Repeated, + pub minute30: Repeated, + pub hour1: Repeated, + pub hour4: Repeated, + pub hour12: Repeated, + pub day3: Last, + pub week1: Last, + pub month1: Last, + pub month3: Last, + pub month6: Last, + pub year1: Last, + pub year10: Last, + pub halving: Last, + pub epoch: Last, +} + +impl UrpdViews +where + T: NumericValue + JsonSchema, +{ + fn new( + name: &str, + source: ReadableBoxedVec, + version: Version, + mappings: &UrpdMappings, + ) -> Self { + Self { + height: repeated(name, source.clone(), version, &mappings.height), + minute10: repeated(name, source.clone(), version, &mappings.minute10), + minute30: repeated(name, source.clone(), version, &mappings.minute30), + hour1: repeated(name, source.clone(), version, &mappings.hour1), + hour4: repeated(name, source.clone(), version, &mappings.hour4), + hour12: repeated(name, source.clone(), version, &mappings.hour12), + day3: last(name, source.clone(), version, &mappings.day3), + week1: last(name, source.clone(), version, &mappings.week1), + month1: last(name, source.clone(), version, &mappings.month1), + month3: last(name, source.clone(), version, &mappings.month3), + month6: last(name, source.clone(), version, &mappings.month6), + year1: last(name, source.clone(), version, &mappings.year1), + year10: last(name, source.clone(), version, &mappings.year10), + halving: last(name, source.clone(), version, &mappings.halving), + epoch: last(name, source, version, &mappings.epoch), + } + } +} + +#[derive(Traversable)] +#[traversable(merge)] +pub struct UrpdMetric +where + T: NumericValue + JsonSchema, +{ + pub day1: StoredDay, + #[traversable(flatten)] + pub views: Box>, +} + +impl UrpdMetric +where + T: NumericValue + JsonSchema, +{ + pub(crate) fn forced_import( + db: &Database, + name: &str, + version: Version, + mappings: &UrpdMappings, + ) -> Result { + let day1 = EagerVec::forced_import(db, name, version)?; + let source = day1.read_only_boxed_clone(); + let views = Box::new(UrpdViews::new(name, source, version, mappings)); + + Ok(Self { day1, views }) + } +} + +fn repeated( + name: &str, + source: ReadableBoxedVec, + version: Version, + mapping: &V, +) -> Repeated +where + I: VecIndex, + T: VecValue, + V: ReadableCloneableVec + ?Sized, +{ + UrpdView::new(name, version, source, mapping.read_only_boxed_clone()) +} + +fn last( + name: &str, + source: ReadableBoxedVec, + version: Version, + mapping: &V, +) -> Last +where + I: VecIndex, + T: VecValue, + V: ReadableCloneableVec + ?Sized, +{ + UrpdView::new(name, version, source, mapping.read_only_boxed_clone()) +} + +pub trait DayStrategy: Send + Sync + 'static { + fn mapping_end(to: usize, mapping_len: usize) -> usize; + fn source_index(mapping: &[Day1], index: usize, source_len: usize) -> Option; +} + +impl DayStrategy for RepeatDay { + fn mapping_end(to: usize, _mapping_len: usize) -> usize { + to + } + + fn source_index(mapping: &[Day1], index: usize, source_len: usize) -> Option { + repeated_source_index(mapping, index, source_len) + } +} + +impl DayStrategy for LastDay { + fn mapping_end(to: usize, mapping_len: usize) -> usize { + to.saturating_add(1).min(mapping_len) + } + + fn source_index(mapping: &[Day1], index: usize, source_len: usize) -> Option { + last_source_index(mapping, index, source_len) + } +} + +pub struct UrpdView +where + I: VecIndex, + T: VecValue, +{ + name: Arc, + version: Version, + source: ReadableBoxedVec, + mapping: ReadableBoxedVec, + _phantom: PhantomData S>, +} + +impl Clone for UrpdView +where + I: VecIndex, + T: VecValue, +{ + fn clone(&self) -> Self { + Self { + name: self.name.clone(), + version: self.version, + source: self.source.clone(), + mapping: self.mapping.clone(), + _phantom: PhantomData, + } + } +} + +impl UrpdView +where + I: VecIndex, + T: VecValue, + S: DayStrategy, +{ + fn new( + name: &str, + version: Version, + source: ReadableBoxedVec, + mapping: ReadableBoxedVec, + ) -> Self { + Self { + name: Arc::from(name), + version, + source, + mapping, + _phantom: PhantomData, + } + } + + fn try_fold_values( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result + where + F: FnMut(B, Option) -> std::result::Result, + { + let mapping_len = self.mapping.len(); + let to = to.min(mapping_len); + if from >= to { + return Ok(init); + } + + let mapping = self + .mapping + .collect_range_dyn(from, S::mapping_end(to, mapping_len)); + let source_len = self.source.len(); + try_fold_mapped( + &*self.source, + 0, + to - from, + |index| S::source_index(&mapping, index, source_len), + init, + f, + ) + } + + fn fold_values(&self, from: usize, to: usize, init: B, mut f: F) -> B + where + F: FnMut(B, Option) -> B, + { + match self.try_fold_values(from, to, init, |acc, value| { + Ok::<_, std::convert::Infallible>(f(acc, value)) + }) { + Ok(result) => result, + Err(error) => match error {}, + } + } +} + +impl AnyVec for UrpdView +where + I: VecIndex, + T: VecValue, + S: DayStrategy, +{ + fn version(&self) -> Version { + self.version + self.source.version() + self.mapping.version() + } + + fn name(&self) -> &str { + &self.name + } + + fn len(&self) -> usize { + self.mapping.len() + } + + fn index_type_to_string(&self) -> &'static str { + I::to_string() + } + + fn region_names(&self) -> Vec { + vec![] + } + + fn value_type_to_size_of(&self) -> usize { + size_of::>() + } + + fn value_type_to_string(&self) -> &'static str { + short_type_name::>() + } +} + +impl TypedVec for UrpdView +where + I: VecIndex, + T: VecValue, + S: DayStrategy, +{ + type I = I; + type T = Option; +} + +impl ReadableVec> for UrpdView +where + I: VecIndex, + T: VecValue, + S: DayStrategy, +{ + fn read_into_at(&self, from: usize, to: usize, buf: &mut Vec>) { + self.fold_values(from, to, (), |(), value| buf.push(value)); + } + + fn for_each_range_dyn_at(&self, from: usize, to: usize, f: &mut dyn FnMut(Option)) { + self.fold_values(from, to, (), |(), value| f(value)); + } + + fn fold_range_at) -> B>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> B { + self.fold_values(from, to, init, f) + } + + fn try_fold_range_at) -> std::result::Result>( + &self, + from: usize, + to: usize, + init: B, + f: F, + ) -> std::result::Result { + self.try_fold_values(from, to, init, f) + } + + fn collect_one_at(&self, index: usize) -> Option> { + let mapping_len = self.mapping.len(); + if index >= mapping_len { + return None; + } + + let mapping = self + .mapping + .collect_range_dyn(index, S::mapping_end(index.saturating_add(1), mapping_len)); + Some( + S::source_index(&mapping, 0, self.source.len()) + .and_then(|day| self.source.collect_one_at(day)), + ) + } +} + +impl Traversable for UrpdView +where + I: VecIndex, + T: NumericValue + JsonSchema, + S: DayStrategy, +{ + fn to_tree_node(&self) -> TreeNode { + let indexes = Index::try_from(I::to_string()).ok().into_iter().collect(); + let leaf = SeriesLeaf::new( + self.name().to_string(), + self.value_type_to_string().to_string(), + indexes, + ); + let schema = schemars::SchemaGenerator::default().into_root_schema_for::>(); + let schema_json = serde_json::to_value(schema).unwrap_or_default(); + + TreeNode::Leaf(SeriesLeafWithSchema::new(leaf, schema_json)) + } + + fn iter_any_exportable(&self) -> impl Iterator { + std::iter::once(self as &dyn AnyExportableVec) + } +} + +fn try_fold_mapped( + source: &S, + from: usize, + to: usize, + mut source_index: G, + init: B, + mut f: F, +) -> std::result::Result +where + T: VecValue, + S: ReadableVec + ?Sized, + F: FnMut(B, Option) -> std::result::Result, + G: FnMut(usize) -> Option, +{ + let mut indices = Vec::with_capacity(to - from); + let mut slots: Vec> = Vec::with_capacity(to - from); + + for output_index in from..to { + let Some(source_index) = source_index(output_index) else { + slots.push(None); + continue; + }; + + let slot = match indices.last() { + Some(&last) if last == source_index => indices.len() - 1, + Some(&last) => { + debug_assert!(last < source_index); + indices.push(source_index); + indices.len() - 1 + } + None => { + indices.push(source_index); + 0 + } + }; + slots.push(Some( + u32::try_from(slot).expect("a Day1 source cannot have more than u32::MAX values"), + )); + } + + let values = source.read_sorted_at(&indices); + slots.into_iter().try_fold(init, |acc, slot| match slot { + Some(slot) => f(acc, Some(values[slot as usize].clone())), + None => f(acc, None), + }) +} + +fn repeated_source_index(mapping: &[Day1], index: usize, source_len: usize) -> Option { + let day = mapping[index].to_usize(); + (day < source_len).then_some(day) +} + +fn last_source_index(mapping: &[Day1], index: usize, source_len: usize) -> Option { + let first = mapping[index].to_usize(); + let next_first = mapping + .get(index + 1) + .map(|day| day.to_usize()) + .unwrap_or(source_len) + .min(source_len); + (first < next_first).then_some(next_first - 1) +} + +#[cfg(test)] +mod tests { + use super::*; + use brk_types::StoredF64; + use vecdb::{AnyStoredVec, WritableVec}; + + #[test] + fn repeat_uses_the_same_daily_value_throughout_the_day() { + let mapping = [Day1::from(0), Day1::from(0), Day1::from(1)]; + + assert_eq!(repeated_source_index(&mapping, 0, 2), Some(0)); + assert_eq!(repeated_source_index(&mapping, 1, 2), Some(0)); + assert_eq!(repeated_source_index(&mapping, 2, 2), Some(1)); + assert_eq!(repeated_source_index(&mapping, 2, 1), None); + } + + #[test] + fn coarser_period_uses_its_last_available_day() { + let mapping = [Day1::from(0), Day1::from(3), Day1::from(6)]; + + assert_eq!(last_source_index(&mapping, 0, 8), Some(2)); + assert_eq!(last_source_index(&mapping, 1, 8), Some(5)); + assert_eq!(last_source_index(&mapping, 2, 8), Some(7)); + assert_eq!(last_source_index(&mapping, 1, 5), Some(4)); + assert_eq!(last_source_index(&mapping, 2, 5), None); + } + + #[test] + fn repeated_view_maps_ranges_and_preserves_missing_days() { + let suffix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = + std::env::temp_dir().join(format!("brk-urpd-view-{}-{suffix}", std::process::id())); + let db = Database::open(&path).unwrap(); + + let mut source: EagerVec> = + EagerVec::forced_import(&db, "source", Version::ONE).unwrap(); + let mut mapping: EagerVec> = + EagerVec::forced_import(&db, "mapping", Version::ONE).unwrap(); + for value in [10.0, 20.0, 30.0] { + source.push(StoredF64::from(value)); + } + for day in [0, 0, 1, 2, 3] { + mapping.push(Day1::from(day)); + } + source.write().unwrap(); + mapping.write().unwrap(); + + let view = UrpdView::::new( + "test", + Version::ONE, + source.read_only_boxed_clone(), + mapping.read_only_boxed_clone(), + ); + + assert_eq!( + view.collect_range_at(0, 5), + vec![ + Some(StoredF64::from(10.0)), + Some(StoredF64::from(10.0)), + Some(StoredF64::from(20.0)), + Some(StoredF64::from(30.0)), + None, + ] + ); + + drop(view); + drop(mapping); + drop(source); + drop(db); + std::fs::remove_dir_all(path).unwrap(); + } +} diff --git a/crates/brk_computer/src/models/bedrock/vecs.rs b/crates/brk_computer/src/models/bedrock/vecs.rs new file mode 100644 index 000000000..ebdad950b --- /dev/null +++ b/crates/brk_computer/src/models/bedrock/vecs.rs @@ -0,0 +1,95 @@ +use brk_traversable::Traversable; +use brk_types::{Dollars, StoredF64}; +use derive_more::{Deref, DerefMut}; +use vecdb::{Database, Rw, StorageMode}; + +use super::urpd_metric::UrpdMetric; + +pub(crate) const MODE_COUNT: usize = 10; + +#[derive(Traversable)] +pub struct Percentiles { + pub pct95: T, + pub pct98: T, + pub pct99: T, + pub pct99_5: T, + pub pct99_9: T, +} + +#[derive(Traversable)] +pub struct Levels { + pub pct10: T, + pub pct20: T, + pub pct30: T, + pub pct40: T, + pub pct50: T, + pub pct60: T, + pub pct70: T, + pub pct80: T, + pub pct90: T, +} + +#[derive(Traversable)] +pub struct ModeVecs { + pub loss_threshold: Percentiles>, + pub floor: Percentiles>, + pub level: Levels>, +} + +#[derive(Traversable)] +pub struct Modes { + pub raw: T, + pub cointime: T, + pub coinflow: T, + pub coinflow_8y: T, + pub coinflow_4y: T, + pub coinflow_2y: T, + pub coinflow_1y: T, + pub coinflow_6m: T, + pub coinflow_3m: T, + pub coinflow_1m: T, +} + +impl Modes { + pub(crate) fn try_from_fn( + mut create: impl FnMut(&'static str) -> Result, + ) -> Result { + Ok(Self { + raw: create("raw")?, + cointime: create("cointime")?, + coinflow: create("coinflow")?, + coinflow_8y: create("coinflow_8y")?, + coinflow_4y: create("coinflow_4y")?, + coinflow_2y: create("coinflow_2y")?, + coinflow_1y: create("coinflow_1y")?, + coinflow_6m: create("coinflow_6m")?, + coinflow_3m: create("coinflow_3m")?, + coinflow_1m: create("coinflow_1m")?, + }) + } + + pub(crate) fn as_mut_array(&mut self) -> [&mut T; MODE_COUNT] { + [ + &mut self.raw, + &mut self.cointime, + &mut self.coinflow, + &mut self.coinflow_8y, + &mut self.coinflow_4y, + &mut self.coinflow_2y, + &mut self.coinflow_1y, + &mut self.coinflow_6m, + &mut self.coinflow_3m, + &mut self.coinflow_1m, + ] + } +} + +#[derive(Deref, DerefMut, Traversable)] +pub struct Vecs { + #[traversable(skip)] + pub(crate) db: Database, + #[deref] + #[deref_mut] + #[traversable(flatten)] + pub modes: Modes>, +} diff --git a/crates/brk_computer/src/models/mod.rs b/crates/brk_computer/src/models/mod.rs new file mode 100644 index 000000000..217c2c041 --- /dev/null +++ b/crates/brk_computer/src/models/mod.rs @@ -0,0 +1 @@ +pub mod bedrock; diff --git a/crates/brk_computer/src/op_return/compute.rs b/crates/brk_computer/src/op_return/compute.rs index 738a2443e..d04587fc2 100644 --- a/crates/brk_computer/src/op_return/compute.rs +++ b/crates/brk_computer/src/op_return/compute.rs @@ -198,10 +198,7 @@ fn compute_data_share( exit: &Exit, ) -> Result<()> { target.compute_binary::>( - max_from, - data, - block_size, - exit, + max_from, data, block_size, exit, ) } diff --git a/crates/brk_computer/src/pools/major.rs b/crates/brk_computer/src/pools/major.rs index 6c48b830b..777ea6635 100644 --- a/crates/brk_computer/src/pools/major.rs +++ b/crates/brk_computer/src/pools/major.rs @@ -8,8 +8,8 @@ use vecdb::{BinaryTransform, Database, Exit, ReadableVec, Rw, StorageMode, Versi use crate::{ blocks, indexes, internal::{ - MaskSats, PercentRollingWindows, RatioU64, ValuePerBlockCumulativeRolling, - WindowStartVec, Windows, + MaskSats, PercentRollingWindows, RatioU64, ValuePerBlockCumulativeRolling, WindowStartVec, + Windows, }, mining, price, }; diff --git a/crates/brk_computer/src/supply/import.rs b/crates/brk_computer/src/supply/import.rs index adb4f46a3..73f76cf4e 100644 --- a/crates/brk_computer/src/supply/import.rs +++ b/crates/brk_computer/src/supply/import.rs @@ -4,7 +4,9 @@ use brk_error::Result; use brk_types::Version; use crate::{ - cointime, distribution, indexes, + distribution, + frameworks::cointime, + indexes, internal::{ LazyFiatPerBlock, LazyRollingDeltasFiatFromHeight, LazyValuePerBlock, PercentPerBlock, RollingWindows, WindowStartVec, Windows, diff --git a/crates/brk_mempool/src/stores/addr_tracker/mod.rs b/crates/brk_mempool/src/stores/addr_tracker/mod.rs index f46cad02b..9ccbf3c53 100644 --- a/crates/brk_mempool/src/stores/addr_tracker/mod.rs +++ b/crates/brk_mempool/src/stores/addr_tracker/mod.rs @@ -141,7 +141,7 @@ impl AddrTracker { #[cfg(test)] mod tests { - use brk_types::{Sats, TxOut}; + use brk_types::{Sats, SatsSigned, TxOut}; use super::*; use crate::test_support::{fake_tx, p2wpkh_script}; @@ -164,6 +164,7 @@ mod tests { let entry = tracker.get(&bytes).expect("addr indexed"); assert_eq!(entry.stats.funded_txo_count, 1); assert_eq!(entry.stats.funded_txo_sum, Sats::from(5_000u64)); + assert_eq!(entry.stats.balance_delta, SatsSigned::from(5_000i64)); assert_eq!(entry.stats.tx_count, 1); let (enters, leaves) = transitions.into_vecs(); @@ -186,6 +187,14 @@ mod tests { let spend = addr_of(&prev_script); tracker.add_tx(&mut transitions, &tx); + assert_eq!( + tracker.get(&recv).expect("receiving addr indexed").stats.balance_delta, + SatsSigned::from(3_500i64), + ); + assert_eq!( + tracker.get(&spend).expect("spending addr indexed").stats.balance_delta, + SatsSigned::from(-4_000i64), + ); tracker.remove_tx(&mut transitions, &tx); assert_eq!(tracker.len(), 0); assert!(tracker.get(&recv).is_none()); diff --git a/crates/brk_query/src/impl/addr/stats.rs b/crates/brk_query/src/impl/addr/stats.rs index 656998ec8..7fa1b1b39 100644 --- a/crates/brk_query/src/impl/addr/stats.rs +++ b/crates/brk_query/src/impl/addr/stats.rs @@ -69,6 +69,7 @@ impl Query { addr, addr_type: output_type, chain_stats: AddrChainStats { + balance: addr_data.received - addr_data.sent, type_index, funded_txo_count: addr_data.funded_txo_count, funded_txo_sum: addr_data.received, diff --git a/crates/brk_types/src/addr_chain_stats.rs b/crates/brk_types/src/addr_chain_stats.rs index 614d0f1b4..ed7585c78 100644 --- a/crates/brk_types/src/addr_chain_stats.rs +++ b/crates/brk_types/src/addr_chain_stats.rs @@ -7,6 +7,9 @@ use serde::{Deserialize, Serialize}; /// Based on mempool.space's format with type_index extension. #[derive(Debug, Serialize, Deserialize, JsonSchema)] pub struct AddrChainStats { + /// Current confirmed balance in satoshis + pub balance: Sats, + /// Total number of transaction outputs that funded this address #[schemars(example = 5)] pub funded_txo_count: u32, diff --git a/crates/brk_types/src/addr_mempool_stats.rs b/crates/brk_types/src/addr_mempool_stats.rs index 682e0d79f..c8fa879df 100644 --- a/crates/brk_types/src/addr_mempool_stats.rs +++ b/crates/brk_types/src/addr_mempool_stats.rs @@ -1,4 +1,4 @@ -use crate::{Sats, TxOut}; +use crate::{Sats, SatsSigned, TxOut}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -9,6 +9,9 @@ use serde::{Deserialize, Serialize}; /// #[derive(Debug, Default, Clone, Hash, Serialize, Deserialize, JsonSchema)] pub struct AddrMempoolStats { + /// Net unconfirmed balance change in satoshis; negative when pending spends exceed receipts + pub balance_delta: SatsSigned, + /// Number of unconfirmed transaction outputs funding this address #[schemars(example = 0)] pub funded_txo_count: u32, @@ -32,21 +35,25 @@ pub struct AddrMempoolStats { impl AddrMempoolStats { pub fn receiving(&mut self, txout: &TxOut) { + self.balance_delta += SatsSigned::from(txout.value); self.funded_txo_count += 1; self.funded_txo_sum += txout.value; } pub fn received(&mut self, txout: &TxOut) { + self.balance_delta -= SatsSigned::from(txout.value); self.funded_txo_count -= 1; self.funded_txo_sum -= txout.value; } pub fn sending(&mut self, txout: &TxOut) { + self.balance_delta -= SatsSigned::from(txout.value); self.spent_txo_count += 1; self.spent_txo_sum += txout.value; } pub fn sent(&mut self, txout: &TxOut) { + self.balance_delta += SatsSigned::from(txout.value); self.spent_txo_count -= 1; self.spent_txo_sum -= txout.value; } diff --git a/crates/brk_types/src/sats_signed.rs b/crates/brk_types/src/sats_signed.rs index 2309af00c..ad22a12a5 100644 --- a/crates/brk_types/src/sats_signed.rs +++ b/crates/brk_types/src/sats_signed.rs @@ -16,6 +16,7 @@ use super::{Bitcoin, Sats}; Debug, PartialEq, Eq, + Hash, PartialOrd, Ord, Clone, diff --git a/modules/brk-client/index.js b/modules/brk-client/index.js index 904aaabb6..2a9c6f250 100644 --- a/modules/brk-client/index.js +++ b/modules/brk-client/index.js @@ -21,6 +21,7 @@ * Based on mempool.space's format with type_index extension. * * @typedef {Object} AddrChainStats + * @property {Sats} balance - Current confirmed balance in satoshis * @property {number} fundedTxoCount - Total number of transaction outputs that funded this address * @property {Sats} fundedTxoSum - Total amount in satoshis received by this address across all funded outputs * @property {number} spentTxoCount - Total number of transaction outputs spent from this address @@ -47,6 +48,7 @@ * Based on mempool.space's format. * * @typedef {Object} AddrMempoolStats + * @property {SatsSigned} balanceDelta - Net unconfirmed balance change in satoshis; negative when pending spends exceed receipts * @property {number} fundedTxoCount - Number of unconfirmed transaction outputs funding this address * @property {Sats} fundedTxoSum - Total amount in satoshis being received in unconfirmed transactions * @property {number} spentTxoCount - Number of unconfirmed transaction inputs spending from this address @@ -3116,6 +3118,39 @@ function createCapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2(client }; } +/** + * @typedef {Object} Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern + * @property {SeriesPattern1} pct10 + * @property {SeriesPattern1} pct20 + * @property {SeriesPattern1} pct30 + * @property {SeriesPattern1} pct40 + * @property {SeriesPattern1} pct50 + * @property {SeriesPattern1} pct60 + * @property {SeriesPattern1} pct70 + * @property {SeriesPattern1} pct80 + * @property {SeriesPattern1} pct90 + */ + +/** + * Create a Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern} + */ +function createPct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern(client, acc) { + return { + pct10: createSeriesPattern1(client, _m(acc, 'pct10')), + pct20: createSeriesPattern1(client, _m(acc, 'pct20')), + pct30: createSeriesPattern1(client, _m(acc, 'pct30')), + pct40: createSeriesPattern1(client, _m(acc, 'pct40')), + pct50: createSeriesPattern1(client, _m(acc, 'pct50')), + pct60: createSeriesPattern1(client, _m(acc, 'pct60')), + pct70: createSeriesPattern1(client, _m(acc, 'pct70')), + pct80: createSeriesPattern1(client, _m(acc, 'pct80')), + pct90: createSeriesPattern1(client, _m(acc, 'pct90')), + }; +} + /** * @typedef {Object} CentsPercentilesRatioRawSatsSmaStdUsdPattern * @property {SeriesPattern1} cents @@ -3233,7 +3268,7 @@ function create_1m1w1y24hPercentRatioRawPattern(client, acc) { * @property {CoindaysCoinyearsDormancyTransferPattern} activity * @property {InMaxMinPerSupplyPattern} costBasis * @property {InPattern} investedCapital - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CapCapitalizedGrossLossMvrvNetPeakPriceProfitSellSoprPattern2} realized * @property {DeltaDominanceHalfInTotalPattern2} supply * @property {CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2} unrealized @@ -3268,6 +3303,35 @@ function createCapLossMvrvNetPriceProfitSoprPattern(client, acc) { }; } +/** + * @typedef {Object} CoindaysLivelinessRatioSupplyVaultednessPattern + * @property {AverageBlockCumulativeSumPattern} coindaysConsumed + * @property {AverageBlockCumulativeSumPattern} coindaysCreated + * @property {AverageBlockCumulativeSumPattern} coindaysStored + * @property {SeriesPattern1} liveliness + * @property {SeriesPattern1} ratio + * @property {ActiveVaultedPattern} supply + * @property {SeriesPattern1} vaultedness + */ + +/** + * Create a CoindaysLivelinessRatioSupplyVaultednessPattern pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {CoindaysLivelinessRatioSupplyVaultednessPattern} + */ +function createCoindaysLivelinessRatioSupplyVaultednessPattern(client, acc) { + return { + coindaysConsumed: createAverageBlockCumulativeSumPattern(client, _m(acc, 'coindays_consumed')), + coindaysCreated: createAverageBlockCumulativeSumPattern(client, _m(acc, 'coindays_created')), + coindaysStored: createAverageBlockCumulativeSumPattern(client, _m(acc, 'coindays_stored')), + liveliness: createSeriesPattern1(client, _m(acc, 'liveliness')), + ratio: createSeriesPattern1(client, _m(acc, 'activity_to_vaultedness')), + supply: createActiveVaultedPattern(client, acc), + vaultedness: createSeriesPattern1(client, _m(acc, 'vaultedness')), + }; +} + /** * @typedef {Object} InMaxMinPerSupplyPattern * @property {PerPattern} inLoss @@ -3388,7 +3452,7 @@ function create_1m1w1y2y4yAllPattern(client, acc) { * @typedef {Object} ActivityAddrOutputsRealizedSupplyUnrealizedPattern * @property {TransferPattern} activity * @property {BaseDeltaPattern} addrCount - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CapLossMvrvPriceProfitPattern} realized * @property {DeltaDominanceTotalPattern} supply * @property {NuplPattern} unrealized @@ -3404,7 +3468,7 @@ function createActivityAddrOutputsRealizedSupplyUnrealizedPattern(client, acc) { return { activity: createTransferPattern(client, _m(acc, 'transfer_volume')), addrCount: createBaseDeltaPattern(client, _m(acc, 'addr_count')), - outputs: createSpendingSpentUnspentPattern(client, acc), + outputs: createSpentUnspentUtxoPattern(client, acc), realized: createCapLossMvrvPriceProfitPattern(client, acc), supply: createDeltaDominanceTotalPattern(client, _m(acc, 'supply')), unrealized: createNuplPattern(client, _m(acc, 'nupl')), @@ -3651,7 +3715,7 @@ function createActiveBidirectionalReactivatedReceivingSendingPattern(client, acc /** * @typedef {Object} ActivityOutputsRealizedSupplyUnrealizedPattern * @property {CoindaysTransferPattern} activity - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CapLossMvrvNetPriceProfitSoprPattern} realized * @property {DeltaDominanceHalfInTotalPattern} supply * @property {LossNetNuplProfitPattern} unrealized @@ -3666,7 +3730,7 @@ function createActiveBidirectionalReactivatedReceivingSendingPattern(client, acc function createActivityOutputsRealizedSupplyUnrealizedPattern(client, acc) { return { activity: createCoindaysTransferPattern(client, acc), - outputs: createSpendingSpentUnspentPattern(client, acc), + outputs: createSpentUnspentUtxoPattern(client, acc), realized: createCapLossMvrvNetPriceProfitSoprPattern(client, acc), supply: createDeltaDominanceHalfInTotalPattern(client, _m(acc, 'supply')), unrealized: createLossNetNuplProfitPattern(client, acc), @@ -3676,7 +3740,7 @@ function createActivityOutputsRealizedSupplyUnrealizedPattern(client, acc) { /** * @typedef {Object} ActivityOutputsRealizedSupplyUnrealizedPattern3 * @property {TransferPattern} activity - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CapLossMvrvPriceProfitPattern} realized * @property {DeltaDominanceHalfInTotalPattern} supply * @property {LossNuplProfitPattern} unrealized @@ -3691,7 +3755,7 @@ function createActivityOutputsRealizedSupplyUnrealizedPattern(client, acc) { function createActivityOutputsRealizedSupplyUnrealizedPattern3(client, acc) { return { activity: createTransferPattern(client, _m(acc, 'transfer_volume')), - outputs: createSpendingSpentUnspentPattern(client, acc), + outputs: createSpentUnspentUtxoPattern(client, acc), realized: createCapLossMvrvPriceProfitPattern(client, acc), supply: createDeltaDominanceHalfInTotalPattern(client, _m(acc, 'supply')), unrealized: createLossNuplProfitPattern(client, acc), @@ -3701,7 +3765,7 @@ function createActivityOutputsRealizedSupplyUnrealizedPattern3(client, acc) { /** * @typedef {Object} ActivityOutputsRealizedSupplyUnrealizedPattern2 * @property {TransferPattern} activity - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CapLossMvrvPriceProfitPattern} realized * @property {DeltaDominanceTotalPattern} supply * @property {NuplPattern} unrealized @@ -3716,7 +3780,7 @@ function createActivityOutputsRealizedSupplyUnrealizedPattern3(client, acc) { function createActivityOutputsRealizedSupplyUnrealizedPattern2(client, acc) { return { activity: createTransferPattern(client, _m(acc, 'transfer_volume')), - outputs: createSpendingSpentUnspentPattern(client, acc), + outputs: createSpentUnspentUtxoPattern(client, acc), realized: createCapLossMvrvPriceProfitPattern(client, acc), supply: createDeltaDominanceTotalPattern(client, _m(acc, 'supply')), unrealized: createNuplPattern(client, _m(acc, 'nupl')), @@ -3773,6 +3837,15 @@ function createBtcCentsDeltaSatsUsdPattern(client, acc) { }; } +/** + * @typedef {Object} BtcCentsInSatsUsdPattern + * @property {SeriesPattern1} btc + * @property {SeriesPattern1} cents + * @property {SharePattern2} inLoss + * @property {SeriesPattern1} sats + * @property {SeriesPattern1} usd + */ + /** * @typedef {Object} BtcCentsSatsShareUsdPattern * @property {SeriesPattern1} btc @@ -3907,6 +3980,33 @@ function createPhsReboundThsPattern(client, acc) { }; } +/** + * @template T + * @typedef {Object} Pct95Pct98Pct99Pattern + * @property {SeriesPattern1} pct95 + * @property {SeriesPattern1} pct98 + * @property {SeriesPattern1} pct99 + * @property {SeriesPattern1} pct995 + * @property {SeriesPattern1} pct999 + */ + +/** + * Create a Pct95Pct98Pct99Pattern pattern node + * @template T + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {Pct95Pct98Pct99Pattern} + */ +function createPct95Pct98Pct99Pattern(client, acc) { + return { + pct95: createSeriesPattern1(client, _m(acc, 'pct95')), + pct98: createSeriesPattern1(client, _m(acc, 'pct98')), + pct99: createSeriesPattern1(client, _m(acc, 'pct99')), + pct995: createSeriesPattern1(client, _m(acc, 'pct99_5')), + pct999: createSeriesPattern1(client, _m(acc, 'pct99_9')), + }; +} + /** * @typedef {Object} _1m1w1y24hPattern4 * @property {BtcCentsSatsUsdPattern} _1m @@ -4352,6 +4452,29 @@ function createLossNetNuplProfitPattern(client, acc) { }; } +/** + * @typedef {Object} MobilitySpendingSupplyPattern + * @property {SeriesPattern1} mobility + * @property {SeriesPattern1} spendingExposure + * @property {SeriesPattern1} spendingRate + * @property {ImmobileMobilePattern} supply + */ + +/** + * Create a MobilitySpendingSupplyPattern pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {MobilitySpendingSupplyPattern} + */ +function createMobilitySpendingSupplyPattern(client, acc) { + return { + mobility: createSeriesPattern1(client, _m(acc, 'mobility')), + spendingExposure: createSeriesPattern1(client, _m(acc, 'spending_exposure')), + spendingRate: createSeriesPattern1(client, _m(acc, 'spending_rate')), + supply: createImmobileMobilePattern(client, acc), + }; +} + /** * @typedef {Object} NuplRealizedSupplyUnrealizedPattern * @property {RatioRawPattern} nupl @@ -4621,6 +4744,27 @@ function createDeltaDominanceTotalPattern(client, acc) { }; } +/** + * @typedef {Object} FloorLevelLossPattern + * @property {Pct95Pct98Pct99Pattern} floor + * @property {Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern} level + * @property {Pct95Pct98Pct99Pattern} lossThreshold + */ + +/** + * Create a FloorLevelLossPattern pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {FloorLevelLossPattern} + */ +function createFloorLevelLossPattern(client, acc) { + return { + floor: createPct95Pct98Pct99Pattern(client, _m(acc, 'floor')), + level: createPct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern(client, _m(acc, 'level')), + lossThreshold: createPct95Pct98Pct99Pattern(client, _m(acc, 'loss_threshold')), + }; +} + /** * @typedef {Object} GreedNetPainPattern * @property {CentsUsdPattern3} greedIndex @@ -4792,23 +4936,23 @@ function createRsiStochPattern(client, acc, disc) { } /** - * @typedef {Object} SpendingSpentUnspentPattern - * @property {SeriesPattern1} spendingRate + * @typedef {Object} SpentUnspentUtxoPattern * @property {AverageBlockCumulativeSumPattern2} spentCount * @property {BaseDeltaPattern} unspentCount + * @property {SeriesPattern1} utxoTurnover1y */ /** - * Create a SpendingSpentUnspentPattern pattern node + * Create a SpentUnspentUtxoPattern pattern node * @param {BrkClient} client * @param {string} acc - Accumulated series name - * @returns {SpendingSpentUnspentPattern} + * @returns {SpentUnspentUtxoPattern} */ -function createSpendingSpentUnspentPattern(client, acc) { +function createSpentUnspentUtxoPattern(client, acc) { return { - spendingRate: createSeriesPattern1(client, _m(acc, 'spending_rate')), spentCount: createAverageBlockCumulativeSumPattern2(client, _m(acc, 'spent_utxo_count')), unspentCount: createBaseDeltaPattern(client, _m(acc, 'utxo_count')), + utxoTurnover1y: createSeriesPattern1(client, _m(acc, 'utxo_turnover_1y')), }; } @@ -4892,6 +5036,25 @@ function createAbsoluteRatePattern3(client, acc) { }; } +/** + * @typedef {Object} ActiveVaultedPattern + * @property {BtcCentsSatsUsdPattern} active + * @property {BtcCentsSatsUsdPattern} vaulted + */ + +/** + * Create a ActiveVaultedPattern pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {ActiveVaultedPattern} + */ +function createActiveVaultedPattern(client, acc) { + return { + active: createBtcCentsSatsUsdPattern(client, _m(acc, 'active_supply')), + vaulted: createBtcCentsSatsUsdPattern(client, _m(acc, 'vaulted_supply')), + }; +} + /** * @typedef {Object} AddrUtxoPattern * @property {BtcCentsSatsUsdPattern} addr @@ -5159,6 +5322,25 @@ function createFundedTotalPattern(client, acc) { }; } +/** + * @typedef {Object} ImmobileMobilePattern + * @property {BtcCentsSatsUsdPattern} immobile + * @property {BtcCentsSatsUsdPattern} mobile + */ + +/** + * Create a ImmobileMobilePattern pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {ImmobileMobilePattern} + */ +function createImmobileMobilePattern(client, acc) { + return { + immobile: createBtcCentsSatsUsdPattern(client, _m(acc, 'immobile_supply')), + mobile: createBtcCentsSatsUsdPattern(client, _m(acc, 'mobile_supply')), + }; +} + /** * @typedef {Object} InPattern2 * @property {CentsUsdPattern3} inLoss @@ -5373,6 +5555,23 @@ function create_24hPattern(client, acc) { }; } +/** + * @typedef {Object} InPattern3 + * @property {SharePattern2} inLoss + */ + +/** + * Create a InPattern3 pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {InPattern3} + */ +function createInPattern3(client, acc) { + return { + inLoss: createSharePattern2(client, acc), + }; +} + /** * @typedef {Object} NuplPattern * @property {RatioRawPattern} nupl @@ -5424,6 +5623,40 @@ function createSharePattern(client, acc) { }; } +/** + * @typedef {Object} SharePattern2 + * @property {SeriesPattern1} share + */ + +/** + * Create a SharePattern2 pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {SharePattern2} + */ +function createSharePattern2(client, acc) { + return { + share: createSeriesPattern1(client, acc), + }; +} + +/** + * @typedef {Object} SupplyPattern + * @property {InPattern3} supply + */ + +/** + * Create a SupplyPattern pattern node + * @param {BrkClient} client + * @param {string} acc - Accumulated series name + * @returns {SupplyPattern} + */ +function createSupplyPattern(client, acc) { + return { + supply: createInPattern3(client, acc), + }; +} + /** * @typedef {Object} TransferPattern * @property {AverageBlockCumulativeSumPattern3} transferVolume @@ -5454,6 +5687,8 @@ function createTransferPattern(client, acc) { * @property {SeriesTree_OpReturn} opReturn * @property {SeriesTree_Mining} mining * @property {SeriesTree_Cointime} cointime + * @property {SeriesTree_Coinflow} coinflow + * @property {SeriesTree_Bedrock} bedrock * @property {SeriesTree_Constants} constants * @property {SeriesTree_Indexes} indexes * @property {SeriesTree_Indicators} indicators @@ -6308,6 +6543,7 @@ function createTransferPattern(client, acc) { /** * @typedef {Object} SeriesTree_Cointime * @property {SeriesTree_Cointime_Activity} activity + * @property {SeriesTree_Cointime_AgeRange} ageRange * @property {SeriesTree_Cointime_Supply} supply * @property {SeriesTree_Cointime_Value} value * @property {SeriesTree_Cointime_Cap} cap @@ -6326,10 +6562,44 @@ function createTransferPattern(client, acc) { * @property {AverageBlockCumulativeSumPattern} coinblocksDestroyed */ +/** + * @typedef {Object} SeriesTree_Cointime_AgeRange + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} under1h + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _1hTo1d + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _1dTo1w + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _1wTo1m + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _1mTo2m + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _2mTo3m + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _3mTo4m + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _4mTo5m + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _5mTo6m + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _6mTo1y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _1yTo2y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _2yTo3y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _3yTo4y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _4yTo5y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _5yTo6y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _6yTo7y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _7yTo8y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _8yTo10y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _10yTo12y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} _12yTo15y + * @property {CoindaysLivelinessRatioSupplyVaultednessPattern} over15y + */ + /** * @typedef {Object} SeriesTree_Cointime_Supply * @property {BtcCentsSatsUsdPattern} vaulted - * @property {BtcCentsSatsUsdPattern} active + * @property {SeriesTree_Cointime_Supply_Active} active + */ + +/** + * @typedef {Object} SeriesTree_Cointime_Supply_Active + * @property {SeriesPattern1} btc + * @property {SeriesPattern1} sats + * @property {SeriesPattern1} usd + * @property {SeriesPattern1} cents + * @property {SharePattern2} inLoss */ /** @@ -6372,6 +6642,80 @@ function createTransferPattern(client, acc) { * @property {SeriesPattern18} hodlBank */ +/** + * @typedef {Object} SeriesTree_Coinflow + * @property {SeriesTree_Coinflow_AgeRange} ageRange + * @property {SeriesTree_Coinflow_Supply} supply + * @property {SeriesTree_Coinflow_Horizon} horizon + * @property {CentsUsdPattern3} cap + * @property {CentsRatioRawSatsUsdPattern} price + */ + +/** + * @typedef {Object} SeriesTree_Coinflow_AgeRange + * @property {MobilitySpendingSupplyPattern} under1h + * @property {MobilitySpendingSupplyPattern} _1hTo1d + * @property {MobilitySpendingSupplyPattern} _1dTo1w + * @property {MobilitySpendingSupplyPattern} _1wTo1m + * @property {MobilitySpendingSupplyPattern} _1mTo2m + * @property {MobilitySpendingSupplyPattern} _2mTo3m + * @property {MobilitySpendingSupplyPattern} _3mTo4m + * @property {MobilitySpendingSupplyPattern} _4mTo5m + * @property {MobilitySpendingSupplyPattern} _5mTo6m + * @property {MobilitySpendingSupplyPattern} _6mTo1y + * @property {MobilitySpendingSupplyPattern} _1yTo2y + * @property {MobilitySpendingSupplyPattern} _2yTo3y + * @property {MobilitySpendingSupplyPattern} _3yTo4y + * @property {MobilitySpendingSupplyPattern} _4yTo5y + * @property {MobilitySpendingSupplyPattern} _5yTo6y + * @property {MobilitySpendingSupplyPattern} _6yTo7y + * @property {MobilitySpendingSupplyPattern} _7yTo8y + * @property {MobilitySpendingSupplyPattern} _8yTo10y + * @property {MobilitySpendingSupplyPattern} _10yTo12y + * @property {MobilitySpendingSupplyPattern} _12yTo15y + * @property {MobilitySpendingSupplyPattern} over15y + */ + +/** + * @typedef {Object} SeriesTree_Coinflow_Supply + * @property {SeriesTree_Coinflow_Supply_Mobile} mobile + * @property {BtcCentsSatsUsdPattern} immobile + */ + +/** + * @typedef {Object} SeriesTree_Coinflow_Supply_Mobile + * @property {SeriesPattern1} btc + * @property {SeriesPattern1} sats + * @property {SeriesPattern1} usd + * @property {SeriesPattern1} cents + * @property {SharePattern2} inLoss + */ + +/** + * @typedef {Object} SeriesTree_Coinflow_Horizon + * @property {SupplyPattern} _8y + * @property {SupplyPattern} _4y + * @property {SupplyPattern} _2y + * @property {SupplyPattern} _1y + * @property {SupplyPattern} _6m + * @property {SupplyPattern} _3m + * @property {SupplyPattern} _1m + */ + +/** + * @typedef {Object} SeriesTree_Bedrock + * @property {FloorLevelLossPattern} raw + * @property {FloorLevelLossPattern} cointime + * @property {FloorLevelLossPattern} coinflow + * @property {FloorLevelLossPattern} coinflow8y + * @property {FloorLevelLossPattern} coinflow4y + * @property {FloorLevelLossPattern} coinflow2y + * @property {FloorLevelLossPattern} coinflow1y + * @property {FloorLevelLossPattern} coinflow6m + * @property {FloorLevelLossPattern} coinflow3m + * @property {FloorLevelLossPattern} coinflow1m + */ + /** * @typedef {Object} SeriesTree_Constants * @property {SeriesPattern1} _0 @@ -7230,7 +7574,7 @@ function createTransferPattern(client, acc) { * @typedef {Object} SeriesTree_Cohorts_Utxo_All_Outputs * @property {BaseDeltaPattern} unspentCount * @property {AverageBlockCumulativeSumPattern2} spentCount - * @property {SeriesPattern1} spendingRate + * @property {SeriesPattern1} utxoTurnover1y */ /** @@ -7425,7 +7769,7 @@ function createTransferPattern(client, acc) { /** * @typedef {Object} SeriesTree_Cohorts_Utxo_Sth * @property {DeltaDominanceHalfInTotalPattern2} supply - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CoindaysCoinyearsDormancyTransferPattern} activity * @property {SeriesTree_Cohorts_Utxo_Sth_Realized} realized * @property {InMaxMinPerSupplyPattern} costBasis @@ -7548,7 +7892,7 @@ function createTransferPattern(client, acc) { /** * @typedef {Object} SeriesTree_Cohorts_Utxo_Lth * @property {DeltaDominanceHalfInTotalPattern2} supply - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CoindaysCoinyearsDormancyTransferPattern} activity * @property {SeriesTree_Cohorts_Utxo_Lth_Realized} realized * @property {InMaxMinPerSupplyPattern} costBasis @@ -7777,7 +8121,7 @@ function createTransferPattern(client, acc) { /** * @typedef {Object} SeriesTree_Cohorts_Utxo_Entry_Discount * @property {DeltaDominanceHalfInTotalPattern2} supply - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CoindaysCoinyearsDormancyTransferPattern} activity * @property {SeriesTree_Cohorts_Utxo_Entry_Discount_Realized} realized * @property {InMaxMinPerSupplyPattern} costBasis @@ -7900,7 +8244,7 @@ function createTransferPattern(client, acc) { /** * @typedef {Object} SeriesTree_Cohorts_Utxo_Entry_Premium * @property {DeltaDominanceHalfInTotalPattern2} supply - * @property {SpendingSpentUnspentPattern} outputs + * @property {SpentUnspentUtxoPattern} outputs * @property {CoindaysCoinyearsDormancyTransferPattern} activity * @property {SeriesTree_Cohorts_Utxo_Entry_Premium_Realized} realized * @property {InMaxMinPerSupplyPattern} costBasis @@ -10084,9 +10428,38 @@ class BrkClient extends BrkClientBase { ratio: createSeriesPattern1(this, 'activity_to_vaultedness'), coinblocksDestroyed: createAverageBlockCumulativeSumPattern(this, 'coinblocks_destroyed'), }, + ageRange: { + under1h: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_under_1h_old'), + _1hTo1d: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_1h_to_1d_old'), + _1dTo1w: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_1d_to_1w_old'), + _1wTo1m: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_1w_to_1m_old'), + _1mTo2m: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_1m_to_2m_old'), + _2mTo3m: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_2m_to_3m_old'), + _3mTo4m: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_3m_to_4m_old'), + _4mTo5m: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_4m_to_5m_old'), + _5mTo6m: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_5m_to_6m_old'), + _6mTo1y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_6m_to_1y_old'), + _1yTo2y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_1y_to_2y_old'), + _2yTo3y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_2y_to_3y_old'), + _3yTo4y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_3y_to_4y_old'), + _4yTo5y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_4y_to_5y_old'), + _5yTo6y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_5y_to_6y_old'), + _6yTo7y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_6y_to_7y_old'), + _7yTo8y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_7y_to_8y_old'), + _8yTo10y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_8y_to_10y_old'), + _10yTo12y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_10y_to_12y_old'), + _12yTo15y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_12y_to_15y_old'), + over15y: createCoindaysLivelinessRatioSupplyVaultednessPattern(this, 'utxos_over_15y_old'), + }, supply: { vaulted: createBtcCentsSatsUsdPattern(this, 'vaulted_supply'), - active: createBtcCentsSatsUsdPattern(this, 'active_supply'), + active: { + btc: createSeriesPattern1(this, 'active_supply'), + sats: createSeriesPattern1(this, 'active_supply_sats'), + usd: createSeriesPattern1(this, 'active_supply_usd'), + cents: createSeriesPattern1(this, 'active_supply_cents'), + inLoss: createSharePattern2(this, 'cointime_supply_in_loss_share'), + }, }, value: { destroyed: createAverageBlockCumulativeSumPattern(this, 'cointime_value_destroyed'), @@ -10119,6 +10492,64 @@ class BrkClient extends BrkClientBase { hodlBank: createSeriesPattern18(this, 'hodl_bank'), }, }, + coinflow: { + ageRange: { + under1h: createMobilitySpendingSupplyPattern(this, 'utxos_under_1h_old'), + _1hTo1d: createMobilitySpendingSupplyPattern(this, 'utxos_1h_to_1d_old'), + _1dTo1w: createMobilitySpendingSupplyPattern(this, 'utxos_1d_to_1w_old'), + _1wTo1m: createMobilitySpendingSupplyPattern(this, 'utxos_1w_to_1m_old'), + _1mTo2m: createMobilitySpendingSupplyPattern(this, 'utxos_1m_to_2m_old'), + _2mTo3m: createMobilitySpendingSupplyPattern(this, 'utxos_2m_to_3m_old'), + _3mTo4m: createMobilitySpendingSupplyPattern(this, 'utxos_3m_to_4m_old'), + _4mTo5m: createMobilitySpendingSupplyPattern(this, 'utxos_4m_to_5m_old'), + _5mTo6m: createMobilitySpendingSupplyPattern(this, 'utxos_5m_to_6m_old'), + _6mTo1y: createMobilitySpendingSupplyPattern(this, 'utxos_6m_to_1y_old'), + _1yTo2y: createMobilitySpendingSupplyPattern(this, 'utxos_1y_to_2y_old'), + _2yTo3y: createMobilitySpendingSupplyPattern(this, 'utxos_2y_to_3y_old'), + _3yTo4y: createMobilitySpendingSupplyPattern(this, 'utxos_3y_to_4y_old'), + _4yTo5y: createMobilitySpendingSupplyPattern(this, 'utxos_4y_to_5y_old'), + _5yTo6y: createMobilitySpendingSupplyPattern(this, 'utxos_5y_to_6y_old'), + _6yTo7y: createMobilitySpendingSupplyPattern(this, 'utxos_6y_to_7y_old'), + _7yTo8y: createMobilitySpendingSupplyPattern(this, 'utxos_7y_to_8y_old'), + _8yTo10y: createMobilitySpendingSupplyPattern(this, 'utxos_8y_to_10y_old'), + _10yTo12y: createMobilitySpendingSupplyPattern(this, 'utxos_10y_to_12y_old'), + _12yTo15y: createMobilitySpendingSupplyPattern(this, 'utxos_12y_to_15y_old'), + over15y: createMobilitySpendingSupplyPattern(this, 'utxos_over_15y_old'), + }, + supply: { + mobile: { + btc: createSeriesPattern1(this, 'mobile_supply'), + sats: createSeriesPattern1(this, 'mobile_supply_sats'), + usd: createSeriesPattern1(this, 'mobile_supply_usd'), + cents: createSeriesPattern1(this, 'mobile_supply_cents'), + inLoss: createSharePattern2(this, 'coinflow_supply_in_loss_share'), + }, + immobile: createBtcCentsSatsUsdPattern(this, 'immobile_supply'), + }, + horizon: { + _8y: createSupplyPattern(this, 'coinflow_8y_supply_in_loss_share'), + _4y: createSupplyPattern(this, 'coinflow_4y_supply_in_loss_share'), + _2y: createSupplyPattern(this, 'coinflow_2y_supply_in_loss_share'), + _1y: createSupplyPattern(this, 'coinflow_1y_supply_in_loss_share'), + _6m: createSupplyPattern(this, 'coinflow_6m_supply_in_loss_share'), + _3m: createSupplyPattern(this, 'coinflow_3m_supply_in_loss_share'), + _1m: createSupplyPattern(this, 'coinflow_1m_supply_in_loss_share'), + }, + cap: createCentsUsdPattern3(this, 'coinflow_cap'), + price: createCentsRatioRawSatsUsdPattern(this, 'coinflow_price'), + }, + bedrock: { + raw: createFloorLevelLossPattern(this, 'bedrock_raw'), + cointime: createFloorLevelLossPattern(this, 'bedrock_cointime'), + coinflow: createFloorLevelLossPattern(this, 'bedrock_coinflow'), + coinflow8y: createFloorLevelLossPattern(this, 'bedrock_coinflow_8y'), + coinflow4y: createFloorLevelLossPattern(this, 'bedrock_coinflow_4y'), + coinflow2y: createFloorLevelLossPattern(this, 'bedrock_coinflow_2y'), + coinflow1y: createFloorLevelLossPattern(this, 'bedrock_coinflow_1y'), + coinflow6m: createFloorLevelLossPattern(this, 'bedrock_coinflow_6m'), + coinflow3m: createFloorLevelLossPattern(this, 'bedrock_coinflow_3m'), + coinflow1m: createFloorLevelLossPattern(this, 'bedrock_coinflow_1m'), + }, constants: { _0: createSeriesPattern1(this, 'constant_0'), _1: createSeriesPattern1(this, 'constant_1'), @@ -10724,7 +11155,7 @@ class BrkClient extends BrkClientBase { outputs: { unspentCount: createBaseDeltaPattern(this, 'utxo_count'), spentCount: createAverageBlockCumulativeSumPattern2(this, 'spent_utxo_count'), - spendingRate: createSeriesPattern1(this, 'spending_rate'), + utxoTurnover1y: createSeriesPattern1(this, 'utxo_turnover_1y'), }, activity: { transferVolume: createAverageBlockCumulativeInSumPattern(this, 'transfer_volume'), @@ -10875,7 +11306,7 @@ class BrkClient extends BrkClientBase { }, sth: { supply: createDeltaDominanceHalfInTotalPattern2(this, 'sth_supply'), - outputs: createSpendingSpentUnspentPattern(this, 'sth'), + outputs: createSpentUnspentUtxoPattern(this, 'sth'), activity: createCoindaysCoinyearsDormancyTransferPattern(this, 'sth'), realized: { cap: createCentsDeltaToUsdPattern(this, 'sth_realized_cap'), @@ -10975,7 +11406,7 @@ class BrkClient extends BrkClientBase { }, lth: { supply: createDeltaDominanceHalfInTotalPattern2(this, 'lth_supply'), - outputs: createSpendingSpentUnspentPattern(this, 'lth'), + outputs: createSpentUnspentUtxoPattern(this, 'lth'), activity: createCoindaysCoinyearsDormancyTransferPattern(this, 'lth'), realized: { cap: createCentsDeltaToUsdPattern(this, 'lth_realized_cap'), @@ -11166,7 +11597,7 @@ class BrkClient extends BrkClientBase { entry: { discount: { supply: createDeltaDominanceHalfInTotalPattern2(this, 'veteran_supply'), - outputs: createSpendingSpentUnspentPattern(this, 'veteran'), + outputs: createSpentUnspentUtxoPattern(this, 'veteran'), activity: createCoindaysCoinyearsDormancyTransferPattern(this, 'veteran'), realized: { cap: createCentsDeltaToUsdPattern(this, 'veteran_realized_cap'), @@ -11266,7 +11697,7 @@ class BrkClient extends BrkClientBase { }, premium: { supply: createDeltaDominanceHalfInTotalPattern2(this, 'rookie_supply'), - outputs: createSpendingSpentUnspentPattern(this, 'rookie'), + outputs: createSpentUnspentUtxoPattern(this, 'rookie'), activity: createCoindaysCoinyearsDormancyTransferPattern(this, 'rookie'), realized: { cap: createCentsDeltaToUsdPattern(this, 'rookie_realized_cap'), diff --git a/packages/brk_client/brk_client/__init__.py b/packages/brk_client/brk_client/__init__.py index d883ea397..1aac9a3c5 100644 --- a/packages/brk_client/brk_client/__init__.py +++ b/packages/brk_client/brk_client/__init__.py @@ -29,6 +29,9 @@ Sats = int TypeIndex = int # Type (P2PKH, P2WPKH, P2SH, P2TR, etc.) OutputType = Literal["p2pk", "p2pk", "p2pkh", "multisig", "p2sh", "op_return", "v0_p2wpkh", "v0_p2wsh", "v1_p2tr", "p2a", "empty", "unknown"] +# Signed satoshis (i64) - for values that can be negative. +# Used for changes, deltas, profit/loss calculations, etc. +SatsSigned = int # Unified index for any address type (funded or empty) AnyAddrIndex = TypeIndex # Bitcoin amount as floating point (1 BTC = 100,000,000 satoshis) @@ -225,9 +228,6 @@ PartsPerMillionSigned64 = int # - $0.001 = 1 sat # - $0.0001 = 0.1 sats (fractional) SatsFract = float -# Signed satoshis (i64) - for values that can be negative. -# Used for changes, deltas, profit/loss calculations, etc. -SatsSigned = int # Version tracking for data schema and computed values. # # Used to detect when stored data needs to be recomputed due to changes @@ -283,6 +283,7 @@ class AddrChainStats(TypedDict): Based on mempool.space's format with type_index extension. Attributes: + balance: Current confirmed balance in satoshis funded_txo_count: Total number of transaction outputs that funded this address funded_txo_sum: Total amount in satoshis received by this address across all funded outputs spent_txo_count: Total number of transaction outputs spent from this address @@ -291,6 +292,7 @@ class AddrChainStats(TypedDict): type_index: Index of this address within its type on the blockchain realized_price: Realized price (average cost basis) in USD """ + balance: Sats funded_txo_count: int funded_txo_sum: Sats spent_txo_count: int @@ -316,12 +318,14 @@ class AddrMempoolStats(TypedDict): Based on mempool.space's format. Attributes: + balance_delta: Net unconfirmed balance change in satoshis; negative when pending spends exceed receipts funded_txo_count: Number of unconfirmed transaction outputs funding this address funded_txo_sum: Total amount in satoshis being received in unconfirmed transactions spent_txo_count: Number of unconfirmed transaction inputs spending from this address spent_txo_sum: Total amount in satoshis being spent in unconfirmed transactions tx_count: Number of unconfirmed transactions involving this address """ + balance_delta: SatsSigned funded_txo_count: int funded_txo_sum: Sats spent_txo_count: int @@ -3270,6 +3274,21 @@ class CapitalizedGrossInvestedLossNetNuplProfitSentimentPattern2: self.profit: CentsToUsdPattern4 = CentsToUsdPattern4(client, _m(acc, 'unrealized_profit')) self.sentiment: GreedNetPainPattern = GreedNetPainPattern(client, acc) +class Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.pct10: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct10')) + self.pct20: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct20')) + self.pct30: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct30')) + self.pct40: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct40')) + self.pct50: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct50')) + self.pct60: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct60')) + self.pct70: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct70')) + self.pct80: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct80')) + self.pct90: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'pct90')) + class CentsPercentilesRatioRawSatsSmaStdUsdPattern: """Pattern struct for repeated tree structure.""" pass @@ -3335,6 +3354,19 @@ class CapLossMvrvNetPriceProfitSoprPattern: self.profit: BlockCumulativeSumPattern = BlockCumulativeSumPattern(client, _m(acc, 'realized_profit')) self.sopr: RatioValuePattern = RatioValuePattern(client, acc) +class CoindaysLivelinessRatioSupplyVaultednessPattern: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.coindays_consumed: AverageBlockCumulativeSumPattern[StoredF64] = AverageBlockCumulativeSumPattern(client, _m(acc, 'coindays_consumed')) + self.coindays_created: AverageBlockCumulativeSumPattern[StoredF64] = AverageBlockCumulativeSumPattern(client, _m(acc, 'coindays_created')) + self.coindays_stored: AverageBlockCumulativeSumPattern[StoredF64] = AverageBlockCumulativeSumPattern(client, _m(acc, 'coindays_stored')) + self.liveliness: SeriesPattern1[StoredF64] = SeriesPattern1(client, _m(acc, 'liveliness')) + self.ratio: SeriesPattern1[StoredF64] = SeriesPattern1(client, _m(acc, 'activity_to_vaultedness')) + self.supply: ActiveVaultedPattern = ActiveVaultedPattern(client, acc) + self.vaultedness: SeriesPattern1[StoredF64] = SeriesPattern1(client, _m(acc, 'vaultedness')) + class InMaxMinPerSupplyPattern: """Pattern struct for repeated tree structure.""" @@ -3393,7 +3425,7 @@ class ActivityAddrOutputsRealizedSupplyUnrealizedPattern: """Create pattern node with accumulated series name.""" self.activity: TransferPattern = TransferPattern(client, _m(acc, 'transfer_volume')) self.addr_count: BaseDeltaPattern = BaseDeltaPattern(client, _m(acc, 'addr_count')) - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, acc) + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, acc) self.realized: CapLossMvrvPriceProfitPattern = CapLossMvrvPriceProfitPattern(client, acc) self.supply: DeltaDominanceTotalPattern = DeltaDominanceTotalPattern(client, _m(acc, 'supply')) self.unrealized: NuplPattern = NuplPattern(client, _m(acc, 'nupl')) @@ -3509,7 +3541,7 @@ class ActivityOutputsRealizedSupplyUnrealizedPattern: def __init__(self, client: BrkClient, acc: str): """Create pattern node with accumulated series name.""" self.activity: CoindaysTransferPattern = CoindaysTransferPattern(client, acc) - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, acc) + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, acc) self.realized: CapLossMvrvNetPriceProfitSoprPattern = CapLossMvrvNetPriceProfitSoprPattern(client, acc) self.supply: DeltaDominanceHalfInTotalPattern = DeltaDominanceHalfInTotalPattern(client, _m(acc, 'supply')) self.unrealized: LossNetNuplProfitPattern = LossNetNuplProfitPattern(client, acc) @@ -3520,7 +3552,7 @@ class ActivityOutputsRealizedSupplyUnrealizedPattern3: def __init__(self, client: BrkClient, acc: str): """Create pattern node with accumulated series name.""" self.activity: TransferPattern = TransferPattern(client, _m(acc, 'transfer_volume')) - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, acc) + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, acc) self.realized: CapLossMvrvPriceProfitPattern = CapLossMvrvPriceProfitPattern(client, acc) self.supply: DeltaDominanceHalfInTotalPattern = DeltaDominanceHalfInTotalPattern(client, _m(acc, 'supply')) self.unrealized: LossNuplProfitPattern = LossNuplProfitPattern(client, acc) @@ -3531,7 +3563,7 @@ class ActivityOutputsRealizedSupplyUnrealizedPattern2: def __init__(self, client: BrkClient, acc: str): """Create pattern node with accumulated series name.""" self.activity: TransferPattern = TransferPattern(client, _m(acc, 'transfer_volume')) - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, acc) + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, acc) self.realized: CapLossMvrvPriceProfitPattern = CapLossMvrvPriceProfitPattern(client, acc) self.supply: DeltaDominanceTotalPattern = DeltaDominanceTotalPattern(client, _m(acc, 'supply')) self.unrealized: NuplPattern = NuplPattern(client, _m(acc, 'nupl')) @@ -3558,6 +3590,10 @@ class BtcCentsDeltaSatsUsdPattern: self.sats: SeriesPattern1[Sats] = SeriesPattern1(client, _m(acc, 'sats')) self.usd: SeriesPattern1[Dollars] = SeriesPattern1(client, _m(acc, 'usd')) +class BtcCentsInSatsUsdPattern: + """Pattern struct for repeated tree structure.""" + pass + class BtcCentsSatsShareUsdPattern: """Pattern struct for repeated tree structure.""" @@ -3617,6 +3653,17 @@ class PhsReboundThsPattern: self.ths: SeriesPattern1[StoredF32] = SeriesPattern1(client, _m(acc, 'ths')) self.ths_min: SeriesPattern1[StoredF32] = SeriesPattern1(client, _m(acc, 'ths_min')) +class Pct95Pct98Pct99Pattern(Generic[T]): + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.pct95: SeriesPattern1[T] = SeriesPattern1(client, _m(acc, 'pct95')) + self.pct98: SeriesPattern1[T] = SeriesPattern1(client, _m(acc, 'pct98')) + self.pct99: SeriesPattern1[T] = SeriesPattern1(client, _m(acc, 'pct99')) + self.pct99_5: SeriesPattern1[T] = SeriesPattern1(client, _m(acc, 'pct99_5')) + self.pct99_9: SeriesPattern1[T] = SeriesPattern1(client, _m(acc, 'pct99_9')) + class _1m1w1y24hPattern4: """Pattern struct for repeated tree structure.""" @@ -3811,6 +3858,16 @@ class LossNetNuplProfitPattern: self.nupl: RatioRawPattern = RatioRawPattern(client, _m(acc, 'nupl')) self.profit: CentsUsdPattern3 = CentsUsdPattern3(client, _m(acc, 'unrealized_profit')) +class MobilitySpendingSupplyPattern: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.mobility: SeriesPattern1[StoredF64] = SeriesPattern1(client, _m(acc, 'mobility')) + self.spending_exposure: SeriesPattern1[StoredF64] = SeriesPattern1(client, _m(acc, 'spending_exposure')) + self.spending_rate: SeriesPattern1[StoredF64] = SeriesPattern1(client, _m(acc, 'spending_rate')) + self.supply: ImmobileMobilePattern = ImmobileMobilePattern(client, acc) + class NuplRealizedSupplyUnrealizedPattern: """Pattern struct for repeated tree structure.""" @@ -3926,6 +3983,15 @@ class DeltaDominanceTotalPattern: self.dominance: PercentRatioRawPattern2 = PercentRatioRawPattern2(client, _m(acc, 'dominance')) self.total: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, acc) +class FloorLevelLossPattern: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.floor: Pct95Pct98Pct99Pattern[Dollars] = Pct95Pct98Pct99Pattern(client, _m(acc, 'floor')) + self.level: Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern = Pct10Pct20Pct30Pct40Pct50Pct60Pct70Pct80Pct90Pattern(client, _m(acc, 'level')) + self.loss_threshold: Pct95Pct98Pct99Pattern[StoredF64] = Pct95Pct98Pct99Pattern(client, _m(acc, 'loss_threshold')) + class GreedNetPainPattern: """Pattern struct for repeated tree structure.""" @@ -3998,14 +4064,14 @@ class RsiStochPattern: self.stoch_rsi_d: PercentRatioRawPattern2 = PercentRatioRawPattern2(client, _m(acc, f'stoch_d_{disc}')) self.stoch_rsi_k: PercentRatioRawPattern2 = PercentRatioRawPattern2(client, _m(acc, f'stoch_k_{disc}')) -class SpendingSpentUnspentPattern: +class SpentUnspentUtxoPattern: """Pattern struct for repeated tree structure.""" def __init__(self, client: BrkClient, acc: str): """Create pattern node with accumulated series name.""" - self.spending_rate: SeriesPattern1[StoredF32] = SeriesPattern1(client, _m(acc, 'spending_rate')) self.spent_count: AverageBlockCumulativeSumPattern2 = AverageBlockCumulativeSumPattern2(client, _m(acc, 'spent_utxo_count')) self.unspent_count: BaseDeltaPattern = BaseDeltaPattern(client, _m(acc, 'utxo_count')) + self.utxo_turnover_1y: SeriesPattern1[StoredF32] = SeriesPattern1(client, _m(acc, 'utxo_turnover_1y')) class _6bBlockTxPattern(Generic[T]): """Pattern struct for repeated tree structure.""" @@ -4040,6 +4106,14 @@ class AbsoluteRatePattern3: self.absolute: _1m1w1y24hPattern7 = _1m1w1y24hPattern7(client, acc) self.rate: _1m1w1y24hPattern2 = _1m1w1y24hPattern2(client, acc) +class ActiveVaultedPattern: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.active: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, _m(acc, 'active_supply')) + self.vaulted: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, _m(acc, 'vaulted_supply')) + class AddrUtxoPattern: """Pattern struct for repeated tree structure.""" @@ -4152,6 +4226,14 @@ class FundedTotalPattern: self.funded: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4 = AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4(client, acc) self.total: AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4 = AllP2aP2pk33P2pk65P2pkhP2shP2trP2wpkhP2wshPattern4(client, _p('total', acc)) +class ImmobileMobilePattern: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.immobile: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, _m(acc, 'immobile_supply')) + self.mobile: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, _m(acc, 'mobile_supply')) + class InPattern2: """Pattern struct for repeated tree structure.""" @@ -4243,6 +4325,13 @@ class _24hPattern: """Create pattern node with accumulated series name.""" self._24h: SeriesPattern1[StoredF64] = SeriesPattern1(client, acc) +class InPattern3: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.in_loss: SharePattern2 = SharePattern2(client, acc) + class NuplPattern: """Pattern struct for repeated tree structure.""" @@ -4264,6 +4353,20 @@ class SharePattern: """Create pattern node with accumulated series name.""" self.share: PercentRatioRawPattern2 = PercentRatioRawPattern2(client, acc) +class SharePattern2: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.share: SeriesPattern1[StoredF64] = SeriesPattern1(client, acc) + +class SupplyPattern: + """Pattern struct for repeated tree structure.""" + + def __init__(self, client: BrkClient, acc: str): + """Create pattern node with accumulated series name.""" + self.supply: InPattern3 = InPattern3(client, acc) + class TransferPattern: """Pattern struct for repeated tree structure.""" @@ -5206,12 +5309,48 @@ class SeriesTree_Cointime_Activity: self.ratio: SeriesPattern1[StoredF64] = SeriesPattern1(client, 'activity_to_vaultedness') self.coinblocks_destroyed: AverageBlockCumulativeSumPattern[StoredF64] = AverageBlockCumulativeSumPattern(client, 'coinblocks_destroyed') +class SeriesTree_Cointime_AgeRange: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self.under_1h: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_under_1h_old') + self._1h_to_1d: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_1h_to_1d_old') + self._1d_to_1w: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_1d_to_1w_old') + self._1w_to_1m: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_1w_to_1m_old') + self._1m_to_2m: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_1m_to_2m_old') + self._2m_to_3m: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_2m_to_3m_old') + self._3m_to_4m: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_3m_to_4m_old') + self._4m_to_5m: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_4m_to_5m_old') + self._5m_to_6m: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_5m_to_6m_old') + self._6m_to_1y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_6m_to_1y_old') + self._1y_to_2y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_1y_to_2y_old') + self._2y_to_3y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_2y_to_3y_old') + self._3y_to_4y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_3y_to_4y_old') + self._4y_to_5y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_4y_to_5y_old') + self._5y_to_6y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_5y_to_6y_old') + self._6y_to_7y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_6y_to_7y_old') + self._7y_to_8y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_7y_to_8y_old') + self._8y_to_10y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_8y_to_10y_old') + self._10y_to_12y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_10y_to_12y_old') + self._12y_to_15y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_12y_to_15y_old') + self.over_15y: CoindaysLivelinessRatioSupplyVaultednessPattern = CoindaysLivelinessRatioSupplyVaultednessPattern(client, 'utxos_over_15y_old') + +class SeriesTree_Cointime_Supply_Active: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self.btc: SeriesPattern1[Bitcoin] = SeriesPattern1(client, 'active_supply') + self.sats: SeriesPattern1[Sats] = SeriesPattern1(client, 'active_supply_sats') + self.usd: SeriesPattern1[Dollars] = SeriesPattern1(client, 'active_supply_usd') + self.cents: SeriesPattern1[Cents] = SeriesPattern1(client, 'active_supply_cents') + self.in_loss: SharePattern2 = SharePattern2(client, 'cointime_supply_in_loss_share') + class SeriesTree_Cointime_Supply: """Series tree node.""" def __init__(self, client: BrkClient, base_path: str = ''): self.vaulted: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, 'vaulted_supply') - self.active: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, 'active_supply') + self.active: SeriesTree_Cointime_Supply_Active = SeriesTree_Cointime_Supply_Active(client) class SeriesTree_Cointime_Value: """Series tree node.""" @@ -5263,6 +5402,7 @@ class SeriesTree_Cointime: def __init__(self, client: BrkClient, base_path: str = ''): self.activity: SeriesTree_Cointime_Activity = SeriesTree_Cointime_Activity(client) + self.age_range: SeriesTree_Cointime_AgeRange = SeriesTree_Cointime_AgeRange(client) self.supply: SeriesTree_Cointime_Supply = SeriesTree_Cointime_Supply(client) self.value: SeriesTree_Cointime_Value = SeriesTree_Cointime_Value(client) self.cap: SeriesTree_Cointime_Cap = SeriesTree_Cointime_Cap(client) @@ -5270,6 +5410,86 @@ class SeriesTree_Cointime: self.adjusted: SeriesTree_Cointime_Adjusted = SeriesTree_Cointime_Adjusted(client) self.reserve_risk: SeriesTree_Cointime_ReserveRisk = SeriesTree_Cointime_ReserveRisk(client) +class SeriesTree_Coinflow_AgeRange: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self.under_1h: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_under_1h_old') + self._1h_to_1d: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_1h_to_1d_old') + self._1d_to_1w: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_1d_to_1w_old') + self._1w_to_1m: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_1w_to_1m_old') + self._1m_to_2m: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_1m_to_2m_old') + self._2m_to_3m: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_2m_to_3m_old') + self._3m_to_4m: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_3m_to_4m_old') + self._4m_to_5m: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_4m_to_5m_old') + self._5m_to_6m: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_5m_to_6m_old') + self._6m_to_1y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_6m_to_1y_old') + self._1y_to_2y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_1y_to_2y_old') + self._2y_to_3y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_2y_to_3y_old') + self._3y_to_4y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_3y_to_4y_old') + self._4y_to_5y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_4y_to_5y_old') + self._5y_to_6y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_5y_to_6y_old') + self._6y_to_7y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_6y_to_7y_old') + self._7y_to_8y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_7y_to_8y_old') + self._8y_to_10y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_8y_to_10y_old') + self._10y_to_12y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_10y_to_12y_old') + self._12y_to_15y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_12y_to_15y_old') + self.over_15y: MobilitySpendingSupplyPattern = MobilitySpendingSupplyPattern(client, 'utxos_over_15y_old') + +class SeriesTree_Coinflow_Supply_Mobile: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self.btc: SeriesPattern1[Bitcoin] = SeriesPattern1(client, 'mobile_supply') + self.sats: SeriesPattern1[Sats] = SeriesPattern1(client, 'mobile_supply_sats') + self.usd: SeriesPattern1[Dollars] = SeriesPattern1(client, 'mobile_supply_usd') + self.cents: SeriesPattern1[Cents] = SeriesPattern1(client, 'mobile_supply_cents') + self.in_loss: SharePattern2 = SharePattern2(client, 'coinflow_supply_in_loss_share') + +class SeriesTree_Coinflow_Supply: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self.mobile: SeriesTree_Coinflow_Supply_Mobile = SeriesTree_Coinflow_Supply_Mobile(client) + self.immobile: BtcCentsSatsUsdPattern = BtcCentsSatsUsdPattern(client, 'immobile_supply') + +class SeriesTree_Coinflow_Horizon: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self._8y: SupplyPattern = SupplyPattern(client, 'coinflow_8y_supply_in_loss_share') + self._4y: SupplyPattern = SupplyPattern(client, 'coinflow_4y_supply_in_loss_share') + self._2y: SupplyPattern = SupplyPattern(client, 'coinflow_2y_supply_in_loss_share') + self._1y: SupplyPattern = SupplyPattern(client, 'coinflow_1y_supply_in_loss_share') + self._6m: SupplyPattern = SupplyPattern(client, 'coinflow_6m_supply_in_loss_share') + self._3m: SupplyPattern = SupplyPattern(client, 'coinflow_3m_supply_in_loss_share') + self._1m: SupplyPattern = SupplyPattern(client, 'coinflow_1m_supply_in_loss_share') + +class SeriesTree_Coinflow: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self.age_range: SeriesTree_Coinflow_AgeRange = SeriesTree_Coinflow_AgeRange(client) + self.supply: SeriesTree_Coinflow_Supply = SeriesTree_Coinflow_Supply(client) + self.horizon: SeriesTree_Coinflow_Horizon = SeriesTree_Coinflow_Horizon(client) + self.cap: CentsUsdPattern3 = CentsUsdPattern3(client, 'coinflow_cap') + self.price: CentsRatioRawSatsUsdPattern = CentsRatioRawSatsUsdPattern(client, 'coinflow_price') + +class SeriesTree_Bedrock: + """Series tree node.""" + + def __init__(self, client: BrkClient, base_path: str = ''): + self.raw: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_raw') + self.cointime: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_cointime') + self.coinflow: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow') + self.coinflow_8y: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow_8y') + self.coinflow_4y: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow_4y') + self.coinflow_2y: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow_2y') + self.coinflow_1y: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow_1y') + self.coinflow_6m: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow_6m') + self.coinflow_3m: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow_3m') + self.coinflow_1m: FloorLevelLossPattern = FloorLevelLossPattern(client, 'bedrock_coinflow_1m') + class SeriesTree_Constants: """Series tree node.""" @@ -6170,7 +6390,7 @@ class SeriesTree_Cohorts_Utxo_All_Outputs: def __init__(self, client: BrkClient, base_path: str = ''): self.unspent_count: BaseDeltaPattern = BaseDeltaPattern(client, 'utxo_count') self.spent_count: AverageBlockCumulativeSumPattern2 = AverageBlockCumulativeSumPattern2(client, 'spent_utxo_count') - self.spending_rate: SeriesPattern1[StoredF32] = SeriesPattern1(client, 'spending_rate') + self.utxo_turnover_1y: SeriesPattern1[StoredF32] = SeriesPattern1(client, 'utxo_turnover_1y') class SeriesTree_Cohorts_Utxo_All_Activity: """Series tree node.""" @@ -6513,7 +6733,7 @@ class SeriesTree_Cohorts_Utxo_Sth: def __init__(self, client: BrkClient, base_path: str = ''): self.supply: DeltaDominanceHalfInTotalPattern2 = DeltaDominanceHalfInTotalPattern2(client, 'sth_supply') - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, 'sth') + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, 'sth') self.activity: CoindaysCoinyearsDormancyTransferPattern = CoindaysCoinyearsDormancyTransferPattern(client, 'sth') self.realized: SeriesTree_Cohorts_Utxo_Sth_Realized = SeriesTree_Cohorts_Utxo_Sth_Realized(client) self.cost_basis: InMaxMinPerSupplyPattern = InMaxMinPerSupplyPattern(client, 'sth') @@ -6644,7 +6864,7 @@ class SeriesTree_Cohorts_Utxo_Lth: def __init__(self, client: BrkClient, base_path: str = ''): self.supply: DeltaDominanceHalfInTotalPattern2 = DeltaDominanceHalfInTotalPattern2(client, 'lth_supply') - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, 'lth') + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, 'lth') self.activity: CoindaysCoinyearsDormancyTransferPattern = CoindaysCoinyearsDormancyTransferPattern(client, 'lth') self.realized: SeriesTree_Cohorts_Utxo_Lth_Realized = SeriesTree_Cohorts_Utxo_Lth_Realized(client) self.cost_basis: InMaxMinPerSupplyPattern = InMaxMinPerSupplyPattern(client, 'lth') @@ -6880,7 +7100,7 @@ class SeriesTree_Cohorts_Utxo_Entry_Discount: def __init__(self, client: BrkClient, base_path: str = ''): self.supply: DeltaDominanceHalfInTotalPattern2 = DeltaDominanceHalfInTotalPattern2(client, 'veteran_supply') - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, 'veteran') + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, 'veteran') self.activity: CoindaysCoinyearsDormancyTransferPattern = CoindaysCoinyearsDormancyTransferPattern(client, 'veteran') self.realized: SeriesTree_Cohorts_Utxo_Entry_Discount_Realized = SeriesTree_Cohorts_Utxo_Entry_Discount_Realized(client) self.cost_basis: InMaxMinPerSupplyPattern = InMaxMinPerSupplyPattern(client, 'veteran') @@ -7011,7 +7231,7 @@ class SeriesTree_Cohorts_Utxo_Entry_Premium: def __init__(self, client: BrkClient, base_path: str = ''): self.supply: DeltaDominanceHalfInTotalPattern2 = DeltaDominanceHalfInTotalPattern2(client, 'rookie_supply') - self.outputs: SpendingSpentUnspentPattern = SpendingSpentUnspentPattern(client, 'rookie') + self.outputs: SpentUnspentUtxoPattern = SpentUnspentUtxoPattern(client, 'rookie') self.activity: CoindaysCoinyearsDormancyTransferPattern = CoindaysCoinyearsDormancyTransferPattern(client, 'rookie') self.realized: SeriesTree_Cohorts_Utxo_Entry_Premium_Realized = SeriesTree_Cohorts_Utxo_Entry_Premium_Realized(client) self.cost_basis: InMaxMinPerSupplyPattern = InMaxMinPerSupplyPattern(client, 'rookie') @@ -7298,6 +7518,8 @@ class SeriesTree: self.op_return: SeriesTree_OpReturn = SeriesTree_OpReturn(client) self.mining: SeriesTree_Mining = SeriesTree_Mining(client) self.cointime: SeriesTree_Cointime = SeriesTree_Cointime(client) + self.coinflow: SeriesTree_Coinflow = SeriesTree_Coinflow(client) + self.bedrock: SeriesTree_Bedrock = SeriesTree_Bedrock(client) self.constants: SeriesTree_Constants = SeriesTree_Constants(client) self.indexes: SeriesTree_Indexes = SeriesTree_Indexes(client) self.indicators: SeriesTree_Indicators = SeriesTree_Indicators(client) diff --git a/website/llms-full.txt b/website/llms-full.txt index 85ed48b34..a0dc98c56 100644 --- a/website/llms-full.txt +++ b/website/llms-full.txt @@ -4,7 +4,7 @@ - Version: `v0.3.6` - Base URL: https://bitview.space -- Metrics: 55667 +- Metrics: 56973 - Operations: 97 For machine-readable tool construction, use [https://bitview.space/openapi.json](https://bitview.space/openapi.json). For the complete source-derived series tree, use [https://bitview.space/api/series](https://bitview.space/api/series). @@ -1450,6 +1450,7 @@ curl -s "https://bitview.space/version" ### `AddrChainStats` +- `balance`: `Sats` (required) — Current confirmed balance in satoshis - `funded_txo_count`: `integer` (required) — Total number of transaction outputs that funded this address - `funded_txo_sum`: `Sats` (required) — Total amount in satoshis received by this address across all funded outputs - `spent_txo_count`: `integer` (required) — Total number of transaction outputs spent from this address @@ -1467,6 +1468,7 @@ curl -s "https://bitview.space/version" ### `AddrMempoolStats` +- `balance_delta`: `SatsSigned` (required) — Net unconfirmed balance change in satoshis; negative when pending spends exceed receipts - `funded_txo_count`: `integer` (required) — Number of unconfirmed transaction outputs funding this address - `funded_txo_sum`: `Sats` (required) — Total amount in satoshis being received in unconfirmed transactions - `spent_txo_count`: `integer` (required) — Number of unconfirmed transaction inputs spending from this address @@ -1908,6 +1910,10 @@ curl -s "https://bitview.space/version" `integer` +### `SatsSigned` + +`integer` + ### `SeriesData` - `version`: `Version` (required) — Version of the series data diff --git a/website/llms.txt b/website/llms.txt index 39c9700fd..8fb278efb 100644 --- a/website/llms.txt +++ b/website/llms.txt @@ -1,6 +1,6 @@ # Bitcoin Research Kit (BRK) -> Free, open-source Bitcoin analytics API and block explorer. 55667 on-chain time-series and 97 API operations. No authentication required. +> Free, open-source Bitcoin analytics API and block explorer. 56973 on-chain time-series and 97 API operations. No authentication required. ## API diff --git a/website_next/ask/compactor.js b/website_next/ask/compactor.js index b22a11c98..e0543119a 100644 --- a/website_next/ask/compactor.js +++ b/website_next/ask/compactor.js @@ -13,10 +13,9 @@ function cancelIdle(scheduled) { /** * @param {Object} options * @param {import("./model.js").AskModel} options.model - * @param {readonly unknown[]} options.tools * @param {(id: string, update: NonNullable>>) => void} options.onCompacted */ -export function createAskCompactor({ model, tools, onCompacted }) { +export function createAskCompactor({ model, onCompacted }) { /** @type {import("./storage.js").StoredChat | undefined} */ let pending; /** @type {number | undefined} */ @@ -37,7 +36,7 @@ export function createAskCompactor({ model, tools, onCompacted }) { if (!target) return; pending = undefined; const controller = new AbortController(); - const promise = compactContext(target, model, tools, controller.signal); + const promise = compactContext(target, model, controller.signal); const task = { controller, promise }; active = task; diff --git a/website_next/ask/context.js b/website_next/ask/context.js index 997e114ad..fdd9fe4a8 100644 --- a/website_next/ask/context.js +++ b/website_next/ask/context.js @@ -55,12 +55,11 @@ function compactionPrompt(memory, messages) { /** * @param {StoredChat} chat * @param {AskModel} model - * @param {readonly unknown[]} tools */ -export async function prepareContext(chat, model, tools) { +export async function prepareContext(chat, model) { let start = chat.compactedCount; let messages = messagesFor(chat, start); - let tokenCount = await model.countTokens(messages, tools); + let tokenCount = await model.countTokens(messages); if (tokenCount <= MAX_INPUT_TOKENS) return { chat, messages }; start = Math.max(start, chat.messages.length - KEEP_RECENT_MESSAGES); @@ -71,7 +70,7 @@ export async function prepareContext(chat, model, tools) { chat.messages[start].role !== "user" ) start += 1; messages = messagesFor(chat, start); - tokenCount = await model.countTokens(messages, tools); + tokenCount = await model.countTokens(messages); } return { chat, messages }; } @@ -79,11 +78,10 @@ export async function prepareContext(chat, model, tools) { /** * @param {StoredChat} chat * @param {AskModel} model - * @param {readonly unknown[]} tools * @param {AbortSignal} signal */ -export async function compactContext(chat, model, tools, signal) { - const tokenCount = await model.countTokens(messagesFor(chat), tools); +export async function compactContext(chat, model, signal) { + const tokenCount = await model.countTokens(messagesFor(chat)); signal.throwIfAborted(); const compactThrough = chat.messages.length - KEEP_RECENT_MESSAGES; if ( diff --git a/website_next/ask/index.js b/website_next/ask/index.js index 34f083945..515741d38 100644 --- a/website_next/ask/index.js +++ b/website_next/ask/index.js @@ -30,7 +30,6 @@ export function createAskPage() { let followingOutput = false; const compactor = createAskCompactor({ model, - tools: assistant.toolsFor(), onCompacted(id, update) { const saved = askStorage.saveMemory(id, update); if (!saved || chat.id !== id) return; @@ -340,11 +339,7 @@ export function createAskPage() { model, async prepare() { timer.set("Preparing context"); - const prepared = await prepareContext( - draft, - model, - assistant.toolsFor(), - ); + const prepared = await prepareContext(draft, model); timer.set("Routing request"); return prepared; }, diff --git a/website_next/ask/storage.js b/website_next/ask/storage.js index a038bb1ec..b1a1d5575 100644 --- a/website_next/ask/storage.js +++ b/website_next/ask/storage.js @@ -51,6 +51,7 @@ const CHART_COLORS = new Set([ * @typedef {Object} ApiContext * @property {string} key * @property {Record} arguments + * @property {string[]} [fields] * * @typedef {Object} SourceContext * @property {string} revision @@ -58,7 +59,6 @@ const CHART_COLORS = new Set([ * @property {number} startLine * @property {number} [endLine] * @property {string} content - * @property {string} [focus] * * @typedef {Object} KnowledgeContext * @property {string} title @@ -194,7 +194,16 @@ function readApiContext(value) { ) .slice(0, 16), ); - return /** @type {ApiContext} */ ({ key: context.key, arguments: arguments_ }); + const fields = Array.isArray(context.fields) + ? context.fields + .filter((field) => typeof field === "string" && field.length <= 256) + .slice(0, 12) + : []; + return /** @type {ApiContext} */ ({ + key: context.key, + arguments: arguments_, + ...(fields.length ? { fields } : {}), + }); } /** @param {unknown} value @returns {SourceContext | undefined} */ @@ -221,9 +230,6 @@ function readSourceContext(value) { startLine: context.startLine, ...(endLine === undefined ? {} : { endLine }), content: context.content.slice(0, 1_000), - ...(typeof context.focus === "string" && context.focus.trim() - ? { focus: context.focus.trim().slice(0, 160) } - : {}), }; } diff --git a/website_next/ask/tools/api/answer.js b/website_next/ask/tools/api/answer.js index da2734b07..586ade085 100644 --- a/website_next/ask/tools/api/answer.js +++ b/website_next/ask/tools/api/answer.js @@ -1,21 +1,41 @@ import { renderApiAnswer } from "../render.js"; -import { normalize } from "../text.js"; +import { relevance } from "../text.js"; import { focusApiData } from "./result.js"; -import { apiRequestWords } from "./routing.js"; -const MAX_FIELDS = 64; +const MAX_FIELDS = 10; + +/** @param {string} type */ +function dimension(type) { + const value = type.toLowerCase(); + if (value.includes("sats")) return "sats"; + return value; +} + +/** @param {string} type */ +function displayedUnit(type) { + const value = dimension(type); + if (value === "sats") return " sats"; + if (value === "number" || value === "integer" || value === "float") return ""; + return ` ${type}`; +} /** - * @typedef {Object} ApiNumericField + * @typedef {Object} ApiAnswerField * @property {string} ref * @property {string} name * @property {string} type * @property {string} [description] - * @property {number} value + * @property {string} [ownDescription] + * @property {string | number | boolean} value + * @property {number} score * * @typedef {Object} ApiAnswerSpec - * @property {ApiNumericField[]} fields - * @property {any} tool + * @property {ApiAnswerField[]} fields + * @property {ApiAnswerField} [previous] + * @property {ApiAnswerField} [resolved] + * @property {ApiAnswerField} [direct] + * @property {ApiAnswerField[]} ambiguous + * @property {any[]} tools */ /** @param {unknown} value @param {string[]} path */ @@ -30,10 +50,21 @@ function valueAt(value, path) { return current; } -/** @param {any} grounding @returns {ApiAnswerSpec} */ -export function createApiAnswerTool(grounding) { +/** @param {unknown} value */ +function formattedValue(value) { + if (typeof value === "number") { + return new Intl.NumberFormat("en-US", { + maximumFractionDigits: 8, + }).format(value); + } + if (typeof value === "boolean") return value ? "yes" : "no"; + return String(value); +} + +/** @param {any} grounding */ +export function summarizeApiAnswer(grounding) { const data = focusApiData(grounding.data, grounding.arguments); - const responseFields = /** @type {{ name: string, type: string, description?: string }[]} */ ( + const responseFields = /** @type {{ name: string, type: string, description?: string, ownDescription?: string }[]} */ ( grounding.operation.response.fields ?? [] ); const fields = responseFields @@ -41,350 +72,383 @@ export function createApiAnswerTool(grounding) { ...field, value: valueAt(data, field.name.split(".")), })) - .filter((field) => typeof field.value === "number") - .slice(0, MAX_FIELDS) + .filter((/** @type {any} */ { value }) => + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) + .slice(0, 8); + if (!fields.length) { + return { + output: renderApiAnswer( + "The API returned no compact primitive fields to display.", + grounding.operation, + ), + fields: [], + }; + } + const output = fields + .map((/** @type {any} */ field) => + `- **${field.name.replaceAll(".", " · ").replaceAll("_", " ")}**: ${ + formattedValue(field.value) + }${typeof field.value === "number" ? displayedUnit(field.type) : ""}` + ) + .join("\n"); + return { + output: renderApiAnswer(output, grounding.operation), + fields: fields.map((/** @type {any} */ { name }) => name), + }; +} + +/** @param {any} grounding @returns {ApiAnswerSpec} */ +export function createApiAnswerTool(grounding) { + const data = focusApiData(grounding.data, grounding.arguments); + const responseFields = /** @type {{ name: string, type: string, description?: string, ownDescription?: string }[]} */ ( + grounding.operation.response.fields ?? [] + ); + const previousName = grounding.previousFields?.length === 1 + ? grounding.previousFields[0] + : undefined; + const previousParents = new Set( + (grounding.previousFields ?? []).map((/** @type {string} */ name) => + name.split(".").slice(0, -1).join(".") + ), + ); + const previousParent = previousParents.size === 1 + ? [...previousParents][0] + : undefined; + const parameterNames = new Set( + grounding.operation.parameters.map( + (/** @type {{ name: string }} */ parameter) => parameter.name, + ), + ); + const primitive = responseFields + .map((field) => ({ + ...field, + value: valueAt(data, field.name.split(".")), + })) + .filter((field) => + typeof field.value === "string" || + typeof field.value === "number" || + typeof field.value === "boolean" + ) .map((field, index) => ({ ...field, - value: /** @type {number} */ (field.value), + index, + score: relevance( + grounding.question, + `${field.name} ${field.description ?? ""}`, + ) + + relevance(grounding.question, field.name) + + relevance(grounding.question, field.ownDescription ?? "") - + Math.max(0, field.name.split(".").length - 1) * 2, + })) + .sort((left, right) => { + const leftParent = left.name.split(".").slice(0, -1).join("."); + const rightParent = right.name.split(".").slice(0, -1).join("."); + const leftAffinity = previousParent && leftParent === previousParent ? 2 : 0; + const rightAffinity = previousParent && rightParent === previousParent ? 2 : 0; + return right.score + rightAffinity - left.score - leftAffinity || + left.index - right.index; + }); + const answerCandidates = primitive.filter(({ name }) => + name !== previousName && + !parameterNames.has(name.split(".").at(-1) ?? name) + ); + const numericCandidates = answerCandidates.filter( + ({ value }) => typeof value === "number", + ); + const best = numericCandidates + .sort((left, right) => right.score - left.score || left.index - right.index)[0]; + const runnerUp = numericCandidates + .filter(({ name }) => name !== best?.name) + .sort((left, right) => right.score - left.score || left.index - right.index)[0]; + const direct = best && best.score >= 6 && + best.score >= (runnerUp?.score ?? 0) + 2 + ? best + : undefined; + const siblings = best + ? primitive.filter((field) => + field.name.split(".").at(-1) === best.name.split(".").at(-1) && + best.score - field.score < 1 && + field.score > 0 + ) + : []; + const matchingParent = previousParent + ? siblings.filter((field) => + field.name.split(".").slice(0, -1).join(".") === previousParent + ) + : []; + const ambiguousNames = new Set( + (matchingParent.length === 1 ? [] : siblings).map(({ name }) => name), + ); + const current = primitive.filter(({ name }) => name !== previousName); + const previousField = previousName + ? primitive.find(({ name }) => name === previousName) + : undefined; + const selected = [ + ...current.slice(0, MAX_FIELDS - (previousField ? 1 : 0)), + ...(previousField ? [previousField] : []), + ]; + const fields = selected + .map((field, index) => ({ + ...field, + value: /** @type {string | number | boolean} */ (field.value), ref: `n${index + 1}`, })); - const fieldDescription = fields - .map((field) => - `${field.ref}=${field.name} (${field.type}): ${field.value}${field.description ? ` — ${field.description}` : ""}` + const previousChoices = fields + .filter(({ name }) => grounding.previousFields?.includes(name)) + .sort((left, right) => right.score - left.score); + const resolved = previousChoices.length > 1 && + previousChoices[0].score >= previousChoices[1].score + 5 + ? previousChoices[0] + : undefined; + const previous = previousName + ? fields.find((field) => + field.name === previousName && typeof field.value === "number" ) - .join("; "); - const canCalculate = fields.length > 0; + : undefined; + const numericFields = fields.filter((field) => typeof field.value === "number"); + const calculationSplit = numericFields.findIndex((field, index) => + index >= 2 && numericFields[index - 1].score - field.score >= 3 + ); + const calculationFields = calculationSplit >= 2 + ? numericFields.slice(0, calculationSplit) + : numericFields; + /** + * @param {string} name + * @param {string} description + * @param {Record} properties + * @param {string[]} required + */ + const functionTool = (name, description, properties, required) => ({ + type: "function", + function: { + name, + description, + parameters: { + type: "object", + properties, + required, + additionalProperties: false, + }, + }, + }); + const label = { + type: "string", + description: "Short user-facing name for the result.", + }; + const operator = { + type: "string", + enum: ["add", "subtract", "multiply", "divide"], + description: "The arithmetic operation explicitly requested by the user.", + }; + const reference = { + type: "string", + enum: fields.map(({ ref }) => ref), + description: "Verified field ref from the user message.", + }; + const numericReference = { + type: "string", + enum: calculationFields.map(({ ref }) => ref), + description: "Verified numeric field ref from the user message.", + }; + const tools = [ + functionTool( + "answer_api", + [ + "Choose select for one raw primitive field.", + "Choose calculate to derive the result from component fields, including a narrower concept than an aggregate.", + previous + ? `Choose continue only to apply arithmetic to preceding ${previous.ref}=${previous.name}.` + : "", + "Choose text for a nonnumeric answer copied or summarized from verified data.", + ].filter(Boolean).join(" "), + { + action: { + type: "string", + enum: [ + ...(fields.length ? ["select"] : []), + ...(calculationFields.length >= 2 ? ["calculate"] : []), + ...(previous ? ["continue"] : []), + "text", + ], + }, + ...(fields.length + ? { + field: reference, + ...(calculationFields.length === 2 + ? { + operator, + left: { + ...numericReference, + description: "Left arithmetic operand: the minuend or dividend for subtract or divide.", + }, + right: { + ...numericReference, + description: "Right arithmetic operand: the subtrahend or divisor for subtract or divide.", + }, + } + : calculationFields.length > 2 + ? { + operator, + operands: { + type: "array", + minItems: 2, + maxItems: 10, + items: numericReference, + description: "Ordered verified numeric fields for calculate.", + }, + } + : {}), + } + : {}), + ...(previous + ? { + operand: { + type: "string", + enum: numericFields + .filter(({ ref }) => ref !== previous.ref) + .map(({ ref }) => ref), + description: `Second operand after fixed ${previous.ref}.`, + }, + } + : {}), + label, + text: { + type: "string", + description: "Concise nonnumeric answer containing no invented values.", + }, + }, + ["action"], + ), + ]; return { fields, - tool: { - type: "function", - function: { - name: "answer_from_api", - description: canCalculate - ? "Answer only from verified API data. Use calculate whenever the requested numeric result combines fields." - : "Answer only from verified API data.", - parameters: { - type: "object", - properties: { - action: { - type: "string", - enum: canCalculate ? ["calculate", "answer"] : ["answer"], - }, - label: { - type: "string", - description: "Short user-facing name for a calculated result.", - }, - ...(canCalculate - ? { - operator: { - type: "string", - enum: ["add", "subtract", "multiply", "divide"], - description: "Arithmetic operator for operands. Use terms instead for a signed sum.", - }, - operands: { - type: "array", - minItems: 2, - maxItems: 12, - items: { - type: "string", - enum: fields.map(({ ref }) => ref), - description: `Verified numeric fields: ${fieldDescription}`, - }, - description: "Ordered source fields for operator arithmetic.", - }, - terms: { - type: "array", - minItems: 1, - maxItems: 12, - items: { - type: "object", - properties: { - ref: { - type: "string", - enum: fields.map(({ ref }) => ref), - description: `Verified numeric fields: ${fieldDescription}`, - }, - sign: { type: "string", enum: ["add", "subtract"] }, - }, - required: ["ref", "sign"], - additionalProperties: false, - }, - description: "Exact arithmetic expression, one signed term per source field.", - }, - } - : {}), - text: { - type: "string", - description: "For answer only: concise answer copied or summarized from verified data, with no invented values.", - }, - }, - required: ["action"], - additionalProperties: false, - }, - }, - }, + previous, + resolved, + direct: direct ? fields.find(({ name }) => name === direct.name) : undefined, + ambiguous: fields.filter(({ name }) => ambiguousNames.has(name)), + tools, }; } -/** @param {string} value */ -function words(value) { - return [...apiRequestWords(value)]; -} - -/** - * @param {string} phrase - * @param {string} context - * @param {ApiNumericField} field - */ -function fieldScore(phrase, context, field) { - const name = normalize(field.name); - const description = normalize(field.description ?? ""); - const document = new Set(words(`${name} ${description}`)); - const phraseWords = words(phrase); - if (!phraseWords.length || !phraseWords.every((word) => document.has(word))) return 0; - - let score = phraseWords.reduce( - (sum, word) => sum + (new Set(words(name)).has(word) ? 8 : 3), - 0, - ); - score += phraseWords.length * 10; - const normalizedPhrase = normalize(phrase); - if (name.includes(normalizedPhrase)) score += 12; - if (description.includes(normalizedPhrase)) score += 5; - for (const word of new Set(words(context))) { - if (document.has(word)) score += name.includes(word) ? 2 : 1; - } - return score; -} - -/** - * Resolve only explicit two-operand subtraction from OpenAPI-derived numeric - * fields. Ambiguous matches fall back to the model. - * - * @param {string} question - * @param {ApiNumericField[]} fields - * @param {any} grounding - */ -export function directApiCalculation(question, fields, grounding) { - const normalizedQuestion = normalize(question); - const virtualSizeRequested = - /\bvsize\b/.test(normalizedQuestion) || - /\bvirtual size\b/.test(normalizedQuestion); - const weight = fields.find((field) => - /\bweight\b/.test(normalize(`${field.name} ${field.type}`)) - ); - const explicitVsize = fields.find((field) => - /\b(?:vsize|virtual size)\b/.test(normalize(`${field.name} ${field.description ?? ""}`)) - ); - if (virtualSizeRequested && (explicitVsize || weight)) { - const value = explicitVsize?.value ?? Math.ceil(/** @type {ApiNumericField} */ (weight).value / 4); +/** @param {string} name @param {Record} action @param {ApiAnswerField[]} fields @param {any} grounding */ +export function finishApiAnswer(name, action, fields, grounding) { + const byRef = new Map(fields.map((field) => [field.ref, field])); + if (name === "select_api_field") { + const field = byRef.get(String(action.field)); + if (!field) throw new Error("The AI selected an unknown API field"); + const label = typeof action.label === "string" && action.label.trim() + ? action.label.trim().replaceAll("_", " ") + : field.name.replaceAll(".", " · ").replaceAll("_", " "); + const formatted = formattedValue(field.value); return renderApiAnswer( - `**virtual size**: ${new Intl.NumberFormat("en-US").format(value)} VSize`, + `**${label}**: ${formatted}${ + typeof field.value === "number" ? displayedUnit(field.type) : "" + }`, grounding.operation, ); } - - const totalNoun = - normalizedQuestion.match(/\btotal\s+([a-z0-9]+)\b/)?.[1] ?? - normalizedQuestion.match(/\b([a-z0-9]+)\s+(?:in\s+)?total\b/)?.[1]; - if (totalNoun) { - const selected = fields.filter((field) => - fieldScore(totalNoun, "total", field) > 0 - ); - if ( - selected.length > 1 && - new Set(selected.map(({ type }) => normalize(type))).size === 1 - ) { - return finishApiAnswer( - { - action: "calculate", - label: `total ${totalNoun}`, - operator: "add", - operands: selected.map(({ ref }) => ref), - }, - fields, - grounding, - ); - } - } - - if (/\bfee\s*rate\b/i.test(question) && /\b(?:imply|derive|calculate|per)\b/i.test(question)) { - const fee = fields.filter((field) => /\bfee\b/.test(normalize(field.name))); - if (fee.length === 1 && explicitVsize) { - return finishApiAnswer( - { - action: "calculate", - label: "fee rate", - operator: "divide", - operands: [fee[0].ref, explicitVsize.ref], - }, - fields, - grounding, - ); - } - if (fee.length === 1 && weight) { - const vsize = Math.ceil(weight.value / 4); - const rate = fee[0].value / vsize; - return renderApiAnswer( - `**fee rate**: ${new Intl.NumberFormat("en-US", { maximumFractionDigits: 8 }).format(rate)} ${fee[0].type}/VSize`, - grounding.operation, - ); - } - } - - /** @type {{ left: string, right: string, context: string } | undefined} */ - let expression; - const cleaned = question.replace(/[?.;]+$/g, "").trim(); - const minus = cleaned.match(/^(.*?)\s+minus\s+(.+)$/i); - if (minus) { - const comma = minus[1].lastIndexOf(","); - const context = comma >= 0 ? minus[1].slice(0, comma) : ""; - const left = (comma >= 0 ? minus[1].slice(comma + 1) : minus[1]).trim(); - expression = { - context, - left, - right: minus[2], - }; - } else { - const difference = cleaned.match( - /^(.*?)\bdifference\s+between\s+(.+?)\s+and\s+(.+)$/i, - ); - if (difference) { - expression = { - context: difference[1], - left: difference[2], - right: difference[3], - }; - } else { - const subtract = cleaned.match( - /^(.*?)\bsubtract\s+(.+?)\s+from\s+(.+)$/i, - ); - if (subtract) { - expression = { - context: subtract[1], - left: subtract[3], - right: subtract[2], - }; - } - } - } - if (!expression) return undefined; - - /** @param {string} value */ - const phraseVariants = (value) => { - const values = words(value); - const phrases = []; - for (let length = 1; length <= Math.min(values.length, 6); length += 1) { - for (let start = 0; start + length <= values.length; start += 1) { - phrases.push(values.slice(start, start + length).join(" ")); - } - } - return phrases; - }; - - /** @param {string} phrase @param {string} context */ - const rank = (phrase, context) => - fields - .map((field) => phraseVariants(phrase) - .map((variant) => ({ - field, - phrase: variant, - score: fieldScore(variant, context, field), - })) - .sort((left, right) => right.score - left.score)[0]) - .filter(({ score }) => score > 0) - .sort((left, right) => right.score - left.score); - - const leftCandidates = rank( - expression.left, - `${expression.context} ${expression.right}`, - ); - const rightCandidates = rank( - expression.right, - `${expression.context} ${expression.left}`, - ); - /** @param {ApiNumericField} field */ - const parent = (field) => field.name.split(".").slice(0, -1).join("."); - const leaf = (/** @type {ApiNumericField} */ field) => field.name.split(".").at(-1); - if (!/\b(?:confirmed|mempool|pending|unconfirmed)\b/i.test(question)) { - const leftLeaf = leftCandidates[0] ? leaf(leftCandidates[0].field) : undefined; - const rightLeaf = rightCandidates[0] ? leaf(rightCandidates[0].field) : undefined; - const leftGroup = leftCandidates.filter(({ field }) => leaf(field) === leftLeaf); - const rightGroup = rightCandidates.filter(({ field }) => leaf(field) === rightLeaf); - if ( - leftGroup.length && - rightGroup.length && - (leftGroup.length > 1 || rightGroup.length > 1) && - new Set( - [...leftGroup, ...rightGroup].map(({ field }) => normalize(field.type)), - ).size === 1 - ) { - return finishApiAnswer( - { - action: "calculate", - label: `${leftGroup[0].phrase} minus ${rightGroup[0].phrase}`, - terms: [ - ...leftGroup.map(({ field }) => ({ ref: field.ref, sign: "add" })), - ...rightGroup.map(({ field }) => ({ ref: field.ref, sign: "subtract" })), - ], - }, - fields, - grounding, - ); - } - } - const pairs = leftCandidates.flatMap((left) => - rightCandidates - .filter((right) => - left.field.ref !== right.field.ref && - normalize(left.field.type) === normalize(right.field.type) - ) - .map((right) => ({ - left, - right, - score: left.score + right.score + - (parent(left.field) === parent(right.field) ? 5 : 0), - })) - ).sort((left, right) => right.score - left.score); - const [pair, second] = pairs; - if (!pair || second?.score === pair.score) return undefined; - - return finishApiAnswer( - { - action: "calculate", - label: `${pair.left.phrase} minus ${pair.right.phrase}`, - terms: [ - { ref: pair.left.field.ref, sign: "add" }, - { ref: pair.right.field.ref, sign: "subtract" }, - ], - }, - fields, - grounding, - ); -} - -/** @param {Record} action @param {ApiNumericField[]} fields @param {any} grounding */ -export function finishApiAnswer(action, fields, grounding) { - if (action.action === "answer") { + if (name === "answer_api_text") { const text = typeof action.text === "string" ? action.text.trim() : ""; if (!text) throw new Error("The AI returned an empty API answer"); return renderApiAnswer(text, grounding.operation); } - if (action.action !== "calculate") { + if (name === "continue_api_calculation") { + const previousNames = Array.isArray(grounding.previousFields) + ? grounding.previousFields + : []; + const previous = previousNames.length === 1 + ? fields.find((field) => field.name === previousNames[0]) + : undefined; + const operand = byRef.get(String(action.operand)); + if ( + !previous || + !operand || + typeof previous.value !== "number" || + typeof operand.value !== "number" || + previous.ref === operand.ref + ) { + throw new Error("The AI returned an invalid API follow-up calculation"); + } + if ( + typeof action.operator !== "string" || + !["add", "subtract", "multiply", "divide"].includes(action.operator) + ) { + throw new Error("The AI returned an invalid API calculation operator"); + } + if (action.operator === "divide" && operand.value === 0) { + throw new Error("Cannot divide by zero"); + } + const value = action.operator === "add" + ? previous.value + operand.value + : action.operator === "subtract" + ? previous.value - operand.value + : action.operator === "multiply" + ? previous.value * operand.value + : previous.value / operand.value; + const previousDimension = dimension(previous.type); + const operandDimension = dimension(operand.type); + const unit = action.operator === "divide" + ? previousDimension === operandDimension + ? "" + : ` ${previousDimension}/${operandDimension}` + : previousDimension === operandDimension + ? displayedUnit(previous.type) + : ""; + const label = typeof action.label === "string" && action.label.trim() + ? action.label.trim().replaceAll("_", " ") + : "result"; + const formatted = new Intl.NumberFormat("en-US", { + maximumFractionDigits: 8, + }).format(value); + return renderApiAnswer(`**${label}**: ${formatted}${unit}`, grounding.operation); + } + if (name !== "calculate_api_fields") { throw new Error("The AI returned an invalid API calculation"); } - const byRef = new Map(fields.map((field) => [field.ref, field])); if ( typeof action.operator === "string" && ["add", "subtract", "multiply", "divide"].includes(action.operator) && - Array.isArray(action.operands) && - action.operands.length >= 2 + ( + ( + typeof action.left === "string" && + typeof action.right === "string" + ) || + ( + Array.isArray(action.operands) && + action.operands.length >= 2 + ) + ) ) { - const selected = action.operands.map((ref) => byRef.get(String(ref))); + const refs = typeof action.left === "string" && + typeof action.right === "string" + ? [action.left, action.right] + : /** @type {unknown[]} */ (action.operands); + const selected = refs.map((ref) => byRef.get(String(ref))); if (selected.some((field) => !field)) throw new Error("Unknown calculation field"); - const numeric = /** @type {ApiNumericField[]} */ (selected); + if (selected.some((field) => typeof field?.value !== "number")) { + throw new Error("A calculation requires numeric fields"); + } + const numeric = /** @type {(ApiAnswerField & { value: number })[]} */ (selected); const [first, ...rest] = numeric; + const types = new Set(numeric.map(({ type }) => dimension(type))); + if ( + (action.operator === "add" || action.operator === "subtract") && + types.size > 1 + ) { + const choices = numeric + .map((field) => + `**${field.name.replaceAll(".", " · ").replaceAll("_", " ")}** (${field.type})` + ) + .join(", "); + return renderApiAnswer( + `Those fields use different units: ${choices}. Which one do you want?`, + grounding.operation, + ); + } const value = rest.reduce((result, field) => { if (action.operator === "add") return result + field.value; if (action.operator === "subtract") return result - field.value; @@ -393,38 +457,17 @@ export function finishApiAnswer(action, fields, grounding) { return result / field.value; }, first.value); const unit = action.operator === "divide" && numeric.length === 2 - ? ` ${first.type}/${numeric[1].type}` - : new Set(numeric.map(({ type }) => type)).size === 1 - ? ` ${first.type}` + ? dimension(first.type) === dimension(numeric[1].type) + ? "" + : ` ${dimension(first.type)}/${dimension(numeric[1].type)}` + : types.size === 1 + ? displayedUnit(first.type) : ""; const label = typeof action.label === "string" && action.label.trim() - ? action.label.trim() + ? action.label.trim().replaceAll("_", " ") : "result"; const formatted = new Intl.NumberFormat("en-US", { maximumFractionDigits: 8 }).format(value); return renderApiAnswer(`**${label}**: ${formatted}${unit}`, grounding.operation); } - if (!Array.isArray(action.terms) || !action.terms.length) { - throw new Error("The AI returned an invalid API calculation"); - } - const selected = action.terms.map((raw) => { - if (!raw || typeof raw !== "object") throw new Error("Invalid calculation term"); - const term = /** @type {Record} */ (raw); - const field = byRef.get(String(term.ref)); - if (!field) throw new Error("Unknown calculation field"); - if (term.sign !== "add" && term.sign !== "subtract") { - throw new Error("Invalid calculation sign"); - } - return { field, sign: term.sign }; - }); - const value = selected.reduce( - (sum, { field, sign }) => sum + (sign === "add" ? field.value : -field.value), - 0, - ); - const types = new Set(selected.map(({ field }) => field.type)); - const unit = types.size === 1 ? ` ${selected[0].field.type}` : ""; - const label = typeof action.label === "string" && action.label.trim() - ? action.label.trim() - : "result"; - const formatted = new Intl.NumberFormat("en-US", { maximumFractionDigits: 8 }).format(value); - return renderApiAnswer(`**${label}**: ${formatted}${unit}`, grounding.operation); + throw new Error("The AI returned an invalid API calculation"); } diff --git a/website_next/ask/tools/api/execute.js b/website_next/ask/tools/api/execute.js index 5e834905f..5d6496306 100644 --- a/website_next/ask/tools/api/execute.js +++ b/website_next/ask/tools/api/execute.js @@ -46,7 +46,7 @@ function compact(value, depth, state) { /** @param {unknown} value @param {import("./index.js").ApiParameter} parameter */ function parameterValue(value, parameter) { - const type = parameter.valueType ?? parameter.type; + const type = parameter.primitive ?? parameter.valueType ?? parameter.type; if (type.includes("integer")) { const number = Number(value); if (!Number.isInteger(number)) throw new Error(`${parameter.name} must be an integer`); diff --git a/website_next/ask/tools/api/index.js b/website_next/ask/tools/api/index.js index 81325f5cc..4e350b846 100644 --- a/website_next/ask/tools/api/index.js +++ b/website_next/ask/tools/api/index.js @@ -11,6 +11,7 @@ const OPENAPI_URL = `${BRK_BASE_URL}/openapi.json`; * @property {boolean} required * @property {string} type * @property {string} [valueType] + * @property {string} [primitive] * @property {string} [format] * @property {unknown[]} [enum] * @property {string} description @@ -26,6 +27,8 @@ const OPENAPI_URL = `${BRK_BASE_URL}/openapi.json`; * @property {{ contentType: string, type: string, description: string, fields: { name: string, type: string, required: boolean, description: string, ownDescription: string }[] }} response * @property {string} [matchedQuery] * @property {number} [matchedTerms] + * @property {number} [titleMatchedTerms] + * @property {number} [specificity] * @property {number} [score] */ diff --git a/website_next/ask/tools/api/openapi.js b/website_next/ask/tools/api/openapi.js index 951ad47c0..8c5da8cc9 100644 --- a/website_next/ask/tools/api/openapi.js +++ b/website_next/ask/tools/api/openapi.js @@ -123,6 +123,7 @@ function parameterDetails(spec, raw) { required: location === "path" || parameter.required === true, type: schemaName(rawSchema), valueType: schemaName(schema), + ...(typeof schema.type === "string" ? { primitive: schema.type } : {}), ...(typeof schema.format === "string" ? { format: schema.format } : {}), ...(Array.isArray(schema.enum) ? { enum: schema.enum.slice(0, 64) } : {}), description: compactText( diff --git a/website_next/ask/tools/api/routing.js b/website_next/ask/tools/api/routing.js index 00ef685ff..4c81c5df1 100644 --- a/website_next/ask/tools/api/routing.js +++ b/website_next/ask/tools/api/routing.js @@ -1,218 +1,9 @@ -import { normalize, tokenAffinity } from "../text.js"; - -/** @param {string} value */ -export function literalArguments(value) { - return [...new Set( - (value.match(/\b(?=[A-Za-z0-9:_-]*\d)[A-Za-z0-9][A-Za-z0-9:_-]*\b/g) ?? []) - .filter((item) => item.length > 1 || /^\d$/.test(item)), - )]; -} - -const API_WORD_ALIASES = new Map([ - ["funded", ["received"]], - ["received", ["funded"]], - ["sent", ["spent"]], - ["spent", ["sent"]], - ["vsize", ["virtual", "size"]], -]); - -/** @param {string} request */ -export function apiRequestWords(request) { - const text = normalize(request); - const words = new Set(text.match(/[a-z0-9]+/g) ?? []); - for (const word of [...words]) { - for (const alias of API_WORD_ALIASES.get(word) ?? []) words.add(alias); - } - if (text.includes("how many")) { - words.add("count"); - words.add("number"); - } - return words; -} - -/** @param {string} request @param {import("./index.js").ApiOperation} operation */ -export function apiResponseMatchCount(request, operation) { - const requestWords = apiRequestWords(request); - const nameWords = new Set(operation.response.fields.flatMap((field) => - normalize(field.name).match(/[a-z0-9]+/g) ?? [] - )); - const descriptionWords = new Set(operation.response.fields.flatMap((field) => - (normalize(field.description).match(/[a-z0-9]+/g) ?? []) - .filter((word) => word.length >= 4) - )); - return [...requestWords].filter((candidate) => - [...nameWords].some((word) => tokenAffinity(word, candidate) >= 0.7) || - candidate.length >= 4 && - [...descriptionWords].some((word) => tokenAffinity(word, candidate) >= 0.65) - ).length; -} - -/** @param {string} request @param {import("./index.js").ApiOperation} operation */ -export function matchesApiResponse(request, operation) { - return apiResponseMatchCount(request, operation) > 0; -} - -/** @param {string} request @param {import("./index.js").ApiOperation} operation */ -export function apiOperationMatchCount(request, operation) { - const requestWords = [...apiRequestWords(request)] - .filter((word) => word.length >= 3); - const operationWords = new Set( - (normalize(`${operation.summary} ${operation.path}`).match(/[a-z0-9]+/g) ?? []) - .filter((word) => word.length >= 3), - ); - return requestWords.filter((candidate) => - [...operationWords].some((word) => - word === candidate || - word === `${candidate}s` || - candidate === `${word}s` || - tokenAffinity(word, candidate) >= 0.7 - ) - ).length; -} - -/** @param {string} request @param {import("./index.js").ApiOperation} operation */ -export function matchesApiIntent(request, operation) { - if (matchesApiResponse(request, operation)) return true; - return apiOperationMatchCount(request, operation) >= 2; -} - -/** @param {string} request */ -function requestsExplanation(request) { - const text = normalize(request); - return /^(?:why|how(?! many\b| much\b))\b/.test(text) || - /\b(?:describe|explain|meaning|mean|works?|working)\b/.test(text) || - /^tell me about\b/.test(text); -} - -/** @param {string} value */ -export function literalType(value) { - if (/^\d{4}-\d{2}-\d{2}$/.test(value)) return "date"; - if (/^-?\d+(?:\.\d+)?$/.test(value) && value.replace(/[^0-9]/g, "").length <= 15) { - return "number"; - } - if (value === "true" || value === "false") return "boolean"; - return "string"; -} - -/** @param {import("./index.js").ApiParameter} parameter */ -function parameterType(parameter) { - if (/^date(?:-time)?$/.test(normalize(parameter.format ?? ""))) return "date"; - const type = normalize(parameter.valueType ?? parameter.type); - if (/\b(?:integer|number)\b/.test(type)) return "number"; - if (/\bboolean\b/.test(type)) return "boolean"; - if (/\bstring\b/.test(type)) return "string"; - return "unknown"; -} - -/** - * @param {string[]} values - * @param {import("./index.js").ApiOperation} operation - * @param {string} [request] - */ -export function argumentAffinity(values, operation, request = "") { - const required = operation.parameters.filter((parameter) => parameter.required); - if (required.length !== values.length) return -1; - if (required.some((parameter, index) => { - const expected = parameterType(parameter); - const actual = literalType(values[index]); - return expected !== "unknown" && expected !== actual; - })) return -1; - const requestTokens = normalize(request).split(" "); - return required.reduce((score, parameter, index) => { - const expected = parameterType(parameter); - const actual = literalType(values[index]); - let next = expected === actual ? score + 2 : expected === "unknown" ? score + 1 : score; - const valueIndex = requestTokens.indexOf(normalize(values[index])); - const context = valueIndex > 0 ? requestTokens[valueIndex - 1] : ""; - const placeholder = `{${parameter.name}}`; - const parts = operation.path.split("/"); - const parameterIndex = parts.indexOf(placeholder); - const pathContext = parameterIndex > 0 ? normalize(parts[parameterIndex - 1]) : ""; - const operationWords = normalize(`${operation.summary} ${operation.path}`) - .match(/[a-z0-9]+/g) ?? []; - if ( - context && - operationWords.some((word) => - word === context || - word === `${context}s` || - context === `${word}s` || - tokenAffinity(word, context) >= 0.7 - ) - ) next += 3; - if ( - context && - pathContext.split(" ").some((word) => - word === context || - word === `${context}s` || - context === `${word}s` - ) - ) next += 2; - return next; - }, 0); -} - -/** - * @param {import("./index.js").ApiOperation[]} hints - * @param {string[]} values - * @param {string} request - */ -export function directApiCandidate(hints, values, request) { - if (requestsExplanation(request)) return undefined; - if (!values.length) { - return hints.find((operation) => - operation.parameters.every((parameter) => !parameter.required) && - ( - apiOperationMatchCount(request, operation) >= 1 || - apiResponseMatchCount(request, operation) >= 2 - ) && - ( - (operation.matchedTerms ?? 0) >= 2 || - (operation.matchedTerms ?? 0) >= 1 && - apiResponseMatchCount(request, operation) >= 1 - ) && - matchesApiIntent(request, operation) - ); - } - const ranked = hints - .map((operation, rank) => ({ - operation, - rank, - affinity: argumentAffinity(values, operation, request), - responseMatches: apiResponseMatchCount(request, operation), - operationMatches: apiOperationMatchCount(request, operation), - })) - .map((candidate) => ({ - ...candidate, - evidence: - candidate.affinity + - candidate.responseMatches * 2 + - candidate.operationMatches * 2, - })) - .filter(({ operation, affinity }) => - affinity >= 0 && (operation.matchedTerms ?? 0) > 0 - ) - .sort((left, right) => - right.evidence - left.evidence || - right.affinity - left.affinity || - right.responseMatches - left.responseMatches || - right.operationMatches - left.operationMatches || - left.rank - right.rank - ); - const [first, second] = ranked; - if (!first || !matchesApiIntent(request, first.operation)) return undefined; - const numericAmbiguity = values.some((value) => literalType(value) === "number") && - second?.evidence === first.evidence && - second.operation.parameters - .filter((parameter) => parameter.required) - .map((parameter) => parameter.name) - .join("|") !== first.operation.parameters - .filter((parameter) => parameter.required) - .map((parameter) => parameter.name) - .join("|"); - return numericAmbiguity ? undefined : first.operation; -} +import { normalize } from "../text.js"; /** + * Reuse only arguments whose names are valid for the selected generated + * OpenAPI operation. No meaning is inferred from the user's wording here. + * * @param {import("./index.js").ApiOperation} operation * @param {{ operation: import("./index.js").ApiOperation, arguments: Record } | undefined} previous */ @@ -229,3 +20,101 @@ export function reusableArguments(operation, previous) { .map((parameter) => [parameter.name, previous.arguments[parameter.name]]), ); } + +/** + * Copy unambiguous primitive values directly from the newest request according + * to the generated OpenAPI parameter schema. This is syntax extraction only; + * operation meaning still comes from generated search metadata and the model. + * + * @param {import("./index.js").ApiOperation} operation + * @param {string} request + */ +export function explicitArguments(operation, request) { + const requiredNumeric = operation.parameters.filter((parameter) => + parameter.required && + (parameter.primitive === "integer" || parameter.primitive === "number") + ); + /** @type {Record} */ + const arguments_ = {}; + if (requiredNumeric.length === 1) { + const matches = request.match( + /(? value.replaceAll(",", "")))]; + if (values.length === 1) { + const value = Number(values[0]); + const parameter = requiredNumeric[0]; + if ( + Number.isFinite(value) && + (parameter.primitive !== "integer" || Number.isInteger(value)) + ) { + arguments_[parameter.name] = values[0]; + } + } + } + const missing = operation.parameters.filter((parameter) => + parameter.required && !Object.hasOwn(arguments_, parameter.name) + ); + if (missing.length === 1 && missing[0].primitive === "string") { + const identifiers = [...new Set( + (request.match(/[A-Za-z0-9][A-Za-z0-9:_-]{11,}/g) ?? []) + .filter((value) => /[A-Za-z]/.test(value) && /\d/.test(value)), + )]; + if (identifiers.length === 1) { + arguments_[missing[0].name] = identifiers[0]; + } + } + for (const parameter of operation.parameters) { + if (Object.hasOwn(arguments_, parameter.name) || !parameter.enum?.length) { + continue; + } + const query = ` ${normalize(request)} `; + const matches = parameter.enum.filter((value) => { + const candidate = normalize(value); + return candidate && query.includes(` ${candidate} `); + }); + if (matches.length === 1) arguments_[parameter.name] = matches[0]; + } + return arguments_; +} + +/** + * @param {import("./index.js").ApiOperation} operation + * @param {Record} arguments_ + */ +export function hasRequiredArguments(operation, arguments_) { + return operation.parameters + .filter((parameter) => parameter.required) + .every((parameter) => Object.hasOwn(arguments_, parameter.name)); +} + +/** + * Keep only generated values that were copied from the newest request or from + * verified context. Source-derived explicit values take precedence. + * + * @param {import("./index.js").ApiOperation} operation + * @param {Record} candidate + * @param {string} request + * @param {{ operation: import("./index.js").ApiOperation, arguments: Record } | undefined} previous + */ +export function validatedArguments(operation, candidate, request, previous) { + const reused = reusableArguments(operation, previous) ?? {}; + const explicit = explicitArguments(operation, request); + const allowed = new Set(operation.parameters.map(({ name }) => name)); + const normalizedRequest = ` ${normalize(request)} `; + const copied = Object.fromEntries( + Object.entries(candidate).filter(([name, value]) => { + if (!allowed.has(name)) return false; + if (Object.hasOwn(reused, name) || Object.hasOwn(explicit, name)) return true; + const raw = String(value); + const normalized = normalize(raw); + return request.includes(raw) || + Boolean(normalized && normalizedRequest.includes(` ${normalized} `)); + }), + ); + return { + ...copied, + ...reused, + ...explicit, + }; +} diff --git a/website_next/ask/tools/api/worker.js b/website_next/ask/tools/api/worker.js index 628bdf0a0..25b25b4e7 100644 --- a/website_next/ask/tools/api/worker.js +++ b/website_next/ask/tools/api/worker.js @@ -26,6 +26,7 @@ function indexOperation(operation) { parameter.name, parameter.type, parameter.valueType, + parameter.primitive, parameter.description, ...(parameter.enum ?? []), ]) @@ -106,15 +107,19 @@ function searchOne(index, query, limit) { const tokens = new Set(operation.tokens); const titleTokens = new Set(operation.titleTokens); let score = 0; + let specificity = 0; let matched = 0; + let titleMatched = 0; for (const word of words) { if (!tokens.has(word)) continue; matched += 1; const frequency = index.documentFrequency.get(word) ?? index.operations.length; const idf = Math.log((index.operations.length + 1) / (frequency + 1)) + 1; + specificity += idf; + if (titleTokens.has(word)) titleMatched += 1; score += idf * (titleTokens.has(word) ? 3 : 1); } - return { operation, matched, score }; + return { operation, matched, titleMatched, score, specificity }; }) .filter(({ matched }) => matched > 0) .sort((left, right) => @@ -125,10 +130,18 @@ function searchOne(index, query, limit) { left.operation.path.localeCompare(right.operation.path) ); if (lexical.length) { - return lexical.slice(0, limit).map(({ operation, matched, score }, rank) => ({ + return lexical.slice(0, limit).map(({ + operation, + matched, + titleMatched, + score, + specificity, + }, rank) => ({ ...publicOperation(operation), matchedQuery: query, matchedTerms: matched, + titleMatchedTerms: titleMatched, + specificity, score: Math.round(score * 1_000) - rank, })); } @@ -164,6 +177,7 @@ function searchOne(index, query, limit) { ), matchedQuery: query, matchedTerms: 0, + titleMatchedTerms: 0, score: 1_000 - rank, })); } diff --git a/website_next/ask/tools/chart.js b/website_next/ask/tools/chart.js index 444a46d11..ff36d844a 100644 --- a/website_next/ask/tools/chart.js +++ b/website_next/ask/tools/chart.js @@ -32,6 +32,14 @@ export function createChartArtifact(args) { throw new Error("A chart needs between one and six series"); } + const usedColors = new Set( + args.series.flatMap((value) => + value && typeof value === "object" && + typeof /** @type {Record} */ (value).color === "string" + ? [/** @type {Record} */ (value).color] + : [] + ), + ); const series = args.series.map((value, index) => { if (!value || typeof value !== "object") throw new Error("Invalid chart series"); const item = /** @type {Record} */ (value); @@ -40,10 +48,11 @@ export function createChartArtifact(args) { .replace(/^series\./, ""); const label = requiredString(item.label, "series label"); const color = item.color === undefined - ? PALETTE[index] + ? PALETTE.find((candidate) => !usedColors.has(candidate)) ?? PALETTE[index] : requiredString(item.color, "series color"); if (!Object.hasOwn(colors, color)) throw new Error(`Unknown chart color: ${color}`); + usedColors.add(color); createMetric(path); return { path, diff --git a/website_next/ask/tools/data.js b/website_next/ask/tools/data.js index b7570d29c..2b7848592 100644 --- a/website_next/ask/tools/data.js +++ b/website_next/ask/tools/data.js @@ -1,19 +1,8 @@ import { brk } from "../../utils/client.js"; +import { unitFromType } from "./metrics/unit.js"; const RANGE_POINTS = 120; -/** @param {string} type */ -export function unitFromType(type) { - const value = type.toLowerCase(); - if (value.includes("dollar") || value.includes("usd") || value.includes("cents")) return "usd"; - if (value.includes("bitcoin") || value === "btc") return "btc"; - if (value.includes("percent") || value.includes("ratio")) return "percent"; - if (value.includes("address")) return "addresses"; - if (value.includes("utxo") || value.includes("output")) return "utxos"; - if (value.includes("block") || value.includes("height")) return "blocks"; - return undefined; -} - /** @param {string} unit */ function displayedUnit(unit) { if (unit === "usd") return "$"; @@ -35,19 +24,23 @@ export function formatValue(value, unit) { /** @param {{ name: string, indexes: string[], type: string, suggestedUnit?: string }} metric @param {Record} action */ export async function readMetric(metric, action) { const rawIndex = typeof action.index === "string" ? action.index : ""; - const indexLooksLikeValue = /^-?\d+$/.test(rawIndex) || /^\d{4}-\d{2}-\d{2}$/.test(rawIndex); - const at = action.at ?? (indexLooksLikeValue ? rawIndex : undefined); + const at = action.at; const dateLike = typeof at === "string" && !/^-?\d+$/.test(at); - const preferredIndex = indexLooksLikeValue ? "" : rawIndex; - const index = metric.indexes.includes(preferredIndex) - ? preferredIndex + const index = metric.indexes.includes(rawIndex) + ? rawIndex : dateLike && metric.indexes.includes("day1") ? "day1" : metric.indexes.includes("height") ? "height" : metric.indexes[0]; if (!index) throw new Error(`No supported index for ${metric.name}`); - const mode = typeof action.mode === "string" ? action.mode : "latest"; + if ( + typeof action.mode !== "string" || + !["latest", "at", "range"].includes(action.mode) + ) { + throw new Error(`A valid position is required for ${metric.name}`); + } + const mode = action.mode; let response; if (mode === "at") { @@ -90,5 +83,6 @@ export async function readMetric(metric, action) { end: response.end, stamp: response.stamp, values: response.data, + ...(mode === "at" ? { requested: String(at) } : {}), }; } diff --git a/website_next/ask/tools/direct/chart.js b/website_next/ask/tools/direct/chart.js deleted file mode 100644 index 42936d5fb..000000000 --- a/website_next/ask/tools/direct/chart.js +++ /dev/null @@ -1,66 +0,0 @@ -import { normalize } from "../text.js"; - -const CHART_REQUEST = - /\b(?:chart|graph|plot|trend|visualize|visualise)\b|\b(?:over|through)\s+time\b|\btime\s+series\b/; -const ADD_REQUEST = /\b(?:add|include|overlay|put)\b/; -const REMOVE_REQUEST = /\b(?:remove|drop)\b|\btake\b.+\boff\b/; -const KEEP_REQUEST = /\b(?:only\s+keep|keep\s+only)\b/; -const UNSUPPORTED_EDIT = /\b(?:clear|replace|reset|swap)\b/; -/** @type {[string, RegExp][]} */ -const VIEW_REQUESTS = [ - ["stacked", /\b(?:stack|stacked)\b/], - ["area", /\barea\b/], - ["bar", /\b(?:bar|bars)\b/], - ["dots", /\b(?:dot|dots|points?)\b/], - ["line", /\b(?:line|lines)\b/], -]; - -/** - * @param {string} request - * @param {boolean} hasActiveChart - */ -export function directChartCommand(request, hasActiveChart) { - const text = normalize(request); - const scale = /\b(?:log|logarithmic)\b/.test(text) - ? "log" - : /\blinear\b/.test(text) - ? "linear" - : undefined; - - if (hasActiveChart) { - const view = VIEW_REQUESTS.find(([, pattern]) => pattern.test(text))?.[0]; - if (scale || view) { - return { - kind: /** @type {const} */ ("style"), - ...(scale ? { scale } : {}), - ...(view ? { view } : {}), - }; - } - } - if (!hasActiveChart && scale) { - return { kind: /** @type {const} */ ("missing") }; - } - if (!hasActiveChart && ADD_REQUEST.test(text) && !CHART_REQUEST.test(text)) { - return { kind: /** @type {const} */ ("missing_add") }; - } - - if (hasActiveChart && ADD_REQUEST.test(text)) { - return { kind: /** @type {const} */ ("edit"), operation: /** @type {const} */ ("add") }; - } - if (hasActiveChart && REMOVE_REQUEST.test(text)) { - return { kind: /** @type {const} */ ("edit"), operation: /** @type {const} */ ("remove") }; - } - if (hasActiveChart && KEEP_REQUEST.test(text)) { - return { kind: /** @type {const} */ ("edit"), operation: /** @type {const} */ ("replace") }; - } - if ( - CHART_REQUEST.test(text) && - !REMOVE_REQUEST.test(text) && - !KEEP_REQUEST.test(text) && - !UNSUPPORTED_EDIT.test(text) && - (!hasActiveChart || !ADD_REQUEST.test(text)) - ) { - return { kind: /** @type {const} */ ("build"), operation: /** @type {const} */ ("add") }; - } - return undefined; -} diff --git a/website_next/ask/tools/direct/evidence.js b/website_next/ask/tools/direct/evidence.js deleted file mode 100644 index a9d8f2b17..000000000 --- a/website_next/ask/tools/direct/evidence.js +++ /dev/null @@ -1,19 +0,0 @@ -import { normalize } from "../text.js"; - -const VARIANTS = /\b(?:availability|available|cohorts?|variants?)\b/; -const IMPLEMENTATION = /\b(?:calculated?|calculation|code|formula|implemented?|implementation|source)\b/; -const PRODUCT = /\b(?:bitview|brk)\b/; -const PRODUCT_QUESTION = /^(?:how|what|where|which|why)\b/; -const DATA_REQUEST = - /\b(?:chart|current|graph|historical|history|latest|now|plot|today|trend|value|visualize|visualise)\b|\b(?:over|through)\s+time\b/; - -/** @param {string} request */ -export function directEvidenceFocus(request) { - const text = normalize(request); - if (IMPLEMENTATION.test(text)) return "implementation"; - if (VARIANTS.test(text)) return "variants"; - if (PRODUCT.test(text) && PRODUCT_QUESTION.test(text) && !DATA_REQUEST.test(text)) { - return "implementation"; - } - return undefined; -} diff --git a/website_next/ask/tools/direct/request.js b/website_next/ask/tools/direct/request.js deleted file mode 100644 index 459f0d071..000000000 --- a/website_next/ask/tools/direct/request.js +++ /dev/null @@ -1,96 +0,0 @@ -import { normalize } from "../text.js"; - -/** @param {string} value */ -export function isExplicitComparison(value) { - return /\b(?:vs\.?|versus|compare|compared|comparison)\b/i.test(value); -} - -/** @param {string} value */ -export function mayRequestMultiple(value) { - return isExplicitComparison(value) || /\b(?:and|both|together)\b/i.test(value); -} - -/** @param {string} value */ -export function referencesPrevious(value) { - return /\b(?:it|its|that|this|they|their|them|those|these|same)\b/i.test(value) || - /^(?:and|also|what about)\b/i.test(value.trim()) || - /^(?:at\s+block\s+\d+|(?:at|on)\s+\d{4}(?:[- ]\d{2}){2})\b/i.test(value.trim()); -} - -/** @param {string} value */ -export function referencesSingular(value) { - return /\b(?:it|its|that|this)\b/i.test(value); -} - -/** @param {string} value */ -export function referencesPlural(value) { - return /\b(?:both|they|their|them|those|these|together)\b/i.test(value); -} - -/** @param {string} request */ -export function isDirectValueFollowup(request) { - const text = normalize(request); - const hasPoint = /\b(?:current|currently|latest|now|today)\b/.test(text) || - /\bblock\s+\d{4,}\b/.test(text) || - /\b\d{4}-\d{2}-\d{2}\b/.test(request); - const needsInterpretation = /^(?:how|why)\b/.test(text) || - /\b(?:available|availability|chart|cohorts?|code|explain|formula|graph|history|plot|source|trend|variants?)\b/.test(text); - return referencesPrevious(text) && hasPoint && !needsInterpretation; -} - -/** @param {string} request */ -export function isDirectValueRequest(request) { - const text = normalize(request); - const hasPoint = /\b(?:current|currently|latest|now|today)\b/.test(text) || - /\bblock\s+\d{4,}\b/.test(text) || - /\b\d{4}-\d{2}-\d{2}\b/.test(request); - const needsDifferentTool = - /\b(?:available|availability|chart|cohorts?|code|explain|formula|graph|history|plot|source|trend|variants?|visualize|visualise)\b/.test(text) || - /\b(?:over|through)\s+time\b/.test(text); - return hasPoint && !needsDifferentTool; -} - -/** @param {string} request @param {string} proposed */ -export function evidenceFocus(request, proposed) { - if (/\b(?:cohorts?|variants?|availability|available)\b/i.test(request)) return "variants"; - if (/\b(?:source|code|implemented?|implementation|calculated?|calculation|formula)\b/i.test(request)) { - return "implementation"; - } - return proposed; -} - -/** @param {string} request */ -export function isDirectDefinition(request) { - const text = normalize(request); - const asks = /\b(?:define|explain|meaning)\b/.test(text) || - /^(?:what is|what are)\b/.test(text) || - /^what does\b.+\bmean\b/.test(text) || - /\bmeans?\s+what\b/.test(text); - const needsRouting = /\b(?:available|availability|chart|cohorts?|code|current|file|graph|history|latest|now|path|plot|source|today|trend|variants?)\b/.test(text) || - /\bblock\s+\d+\b/.test(text) || - /\b\d{4}-\d{2}-\d{2}\b/.test(request); - return asks && !needsRouting; -} - -/** @param {string} request */ -export function directReadAction(request) { - const block = request.match(/\bblock\s+(\d{4,})\b/i)?.[1]; - const dates = [...request.matchAll(/\b\d{4}-\d{2}-\d{2}\b/g)].map(([date]) => date); - if (block) return { mode: "at", index: "height", at: block }; - if (dates.length === 1) return { mode: "at", index: "day1", at: dates[0] }; - if (dates.length > 1 || /\b(?:ago|before|after|between|from|last|previous|since|yesterday)\b/i.test(request)) { - return undefined; - } - return { mode: "latest" }; -} - -/** @param {string[]} queries */ -export function completeComparisonQueries(queries) { - if (queries.length < 3) return queries; - - const shared = queries.at(-1) ?? ""; - const qualifiers = queries.slice(0, -1); - if (!qualifiers.every((query) => normalize(query).split(" ").length === 1)) return queries; - - return qualifiers.map((qualifier) => `${qualifier} ${shared}`); -} diff --git a/website_next/ask/tools/history.js b/website_next/ask/tools/history.js deleted file mode 100644 index 0c3e916e6..000000000 --- a/website_next/ask/tools/history.js +++ /dev/null @@ -1,79 +0,0 @@ -/** @param {any} message */ -function latestMessageChart(message) { - return message.artifacts?.findLast?.( - (/** @type {any} */ artifact) => artifact.type === "chart", - ); -} - -/** @param {any[]} history */ -export function latestChart(history) { - for (const message of [...history].reverse()) { - const chart = latestMessageChart(message); - if (chart) return chart; - } - return undefined; -} - -/** @param {any[]} history @returns {string[] | undefined} */ -export function latestMetricPaths(history) { - for (const message of [...history].reverse()) { - if (Array.isArray(message.metricPaths)) return message.metricPaths; - const chart = latestMessageChart(message); - if (chart) return chart.chart.series.map((/** @type {any} */ item) => item.path); - } - return undefined; -} - -/** @param {any[]} history */ -export function recentMetricPaths(history) { - /** @type {string[]} */ - const paths = []; - for (const message of [...history].reverse()) { - const chart = latestMessageChart(message); - const remembered = Array.isArray(message.metricPaths) - ? message.metricPaths - : chart?.chart.series.map((/** @type {any} */ item) => item.path); - for (const path of remembered ?? []) { - if (!paths.includes(path)) paths.push(path); - if (paths.length === 6) return paths; - } - } - return paths; -} - -/** @param {any[]} history */ -export function latestApiContext(history) { - for (const message of [...history].reverse()) { - if (message.apiContext) return message.apiContext; - if (Array.isArray(message.metricPaths) || latestMessageChart(message)) return undefined; - } - return undefined; -} - -/** @param {any[]} history */ -export function latestSourceContext(history) { - for (const message of [...history].reverse()) { - if (Array.isArray(message.sourceContext) && message.sourceContext.length) { - return message.sourceContext; - } - if ( - message.apiContext || - Array.isArray(message.metricPaths) || - latestMessageChart(message) - ) return undefined; - } - return undefined; -} - -/** @param {any[]} history */ -export function latestKnowledgeContext(history) { - for (const message of [...history].reverse()) { - if (message.knowledgeContext) return message.knowledgeContext; - if ( - message.apiContext || - Array.isArray(message.sourceContext) && message.sourceContext.length || - latestMessageChart(message) - ) return undefined; - } - return undefined; -} diff --git a/website_next/ask/tools/index.js b/website_next/ask/tools/index.js index 644f0dd8e..43bb83699 100644 --- a/website_next/ask/tools/index.js +++ b/website_next/ask/tools/index.js @@ -1,93 +1,132 @@ -import { searchTool } from "./schemas.js"; -import { AskToolSession } from "./session.js"; -import { AskSource } from "./source/index.js"; import { createApiAnswerTool, - directApiCalculation, finishApiAnswer, + summarizeApiAnswer, } from "./api/answer.js"; import { prewarmApiIndex, terminateApiIndex } from "./api/index.js"; import { prewarmMetricIndex, terminateMetricIndex } from "./metrics/index.js"; -import { renderDirectApiAnswer, renderEvidence } from "./render.js"; -import { directSourceFact } from "./source/answer.js"; +import { renderEvidence } from "./render.js"; +import { AskToolSession } from "./session/index.js"; +import { AskSource } from "./source/index.js"; +import { normalize } from "./text.js"; -const MAX_TOOL_ROUNDS = 8; -const GATE_PROMPT = `You are the front door for Bitview's local Bitcoin assistant. -Respond directly in at most 60 words when ordinary Bitcoin knowledge, conversation, or writing is enough. -For follow-ups, keep the subject established by the recent conversation unless the user clearly changes it. Never silently substitute a different Bitcoin topic. -Separate protocol facts from interpretations. Never claim a Bitcoin mechanism guarantees price, value, security, identity, or fraud detection. Do not say fixed supply makes price stable or prevents all inflation. On-chain records identify scripts and addresses, not verified people. -When an essential subject or previous topic is missing, output only one clarification question of at most 15 words. Never guess it, explain possibilities, or list examples. -If and only if the request needs current or historical Bitview data, a concrete public blockchain record, server/API state, metric lookup, charts, cohorts, variants, or BRK repository evidence, return exactly: -TOOLS -BRK is software, not a cryptocurrency or token. +const NUMBER = /\d+(?:[.,]\d+)*/g; -Examples: -User: Which holder group? -Assistant: Which metric or Bitcoin concept do you mean? -User: What can you do? -Assistant: I can explain Bitcoin concepts, search current BRK source, read Bitview metrics and blockchain records, and build charts. Everything runs locally in your browser. -User: Why does Bitcoin have a fixed supply? -Assistant: Bitcoin's consensus rules cap issuance at 21 million BTC. The block subsidy halves roughly every four years, so new issuance declines until the cap is approached. -User: Tell me something about on-chain data. -Assistant: On-chain data is the public record written to Bitcoin's blockchain, including transactions, amounts, fees, and block details. Analysts aggregate it to study network activity and holder behavior. -User: Chart capitalized price. -Assistant: TOOLS -User: What fee did transaction 4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b pay? -Assistant: TOOLS`; - -/** @param {import("../model.js").AskModel} model @param {import("../model.js").ChatMessage[]} messages */ -async function useFrontDoor(model, messages) { - const [, ...dialogue] = messages; - const result = await model.generate( - [ - { - role: "system", - content: GATE_PROMPT, - }, - ...dialogue, - ], - () => {}, - [], - "none", - { maxTokens: 96 }, - ); - const text = result.text.trim(); - return text === "TOOLS" - ? { kind: "tools" } - : { kind: "answer", text }; +/** @param {unknown} value */ +function numbers(value) { + return String(value) + .match(NUMBER) + ?.map((number) => number.replaceAll(",", "")) ?? []; } -/** @param {AskToolSession} session */ -function modelStatus(session) { - if (session.stage === "rewrite") return "Refining search…"; - if (session.stage === "resolve") { - if (session.outcome === "read_api") return "Selecting API…"; - if (session.outcome === "explain_from_verified_facts") { - return session.options.some((option) => option.kind === "source") - ? "Selecting source…" - : "Selecting evidence…"; - } - return "Selecting metrics…"; - } - return "Understanding request…"; +/** + * Quantities in an ungrounded answer are worth a second look. This deliberately + * checks syntax rather than guessing the user's intent or maintaining a list of + * Bitcoin concepts. + * + * @param {string} answer + * @param {import("../model.js").ChatMessage[]} messages + */ +function unsupportedNumbers(answer, messages) { + const supported = new Set(numbers( + messages + .filter(({ role }) => role === "user") + .map(({ content }) => content) + .join(" "), + )); + return new Set(numbers(answer).filter((number) => !supported.has(number))); +} + +/** + * The small model can repeat an unsupported quantity after being asked to + * revise it. Keep the useful grounded sentences instead of trusting a second + * model pass to police itself. + * + * @param {string} answer + * @param {import("../model.js").ChatMessage[]} messages + */ +function removeUnsupportedQuantitySentences(answer, messages) { + const unsupported = unsupportedNumbers(answer, messages); + if (!unsupported.size) return answer.trim(); + + const kept = [...new Intl.Segmenter(undefined, { granularity: "sentence" }) + .segment(answer)] + .map(({ segment }) => segment.trim()) + .filter((segment) => + !numbers(segment).some((number) => unsupported.has(number)) + ); + return kept.join(" ").trim(); } /** * @typedef {Object} ToolOutcome * @property {boolean} done - * @property {boolean} [general] * @property {string} [output] * @property {import("../storage.js").StoredArtifact[]} [artifacts] * @property {string[]} [metricPaths] - * @property {{ key: string, arguments: Record }} [apiContext] + * @property {import("../storage.js").ApiContext} [apiContext] * @property {import("../storage.js").SourceContext[]} [sourceContext] * @property {import("../storage.js").KnowledgeContext} [knowledgeContext] - * @property {{ question: string, metric?: { name: string, path: string, unit?: string }, excerpts: { revision: string, path: string, startLine: number, endLine?: number, content: string }[] }} [grounding] - * @property {{ question: string, context: import("../storage.js").KnowledgeContext }} [knowledgeGrounding] - * @property {{ question: string, operation: { key: string, method: string, path: string, summary: string, description: string, parameters: { name: string }[], response: { fields?: { name: string, type: string, description?: string }[] } }, arguments: Record, requestPath: string, data: unknown, truncated: boolean }} [apiGrounding] - * @property {NonNullable[]} [apiGroundings] + * @property {{ question: string, metrics: { name: string, path: string, unit?: string }[], facts: string[], excerpts: import("../storage.js").SourceContext[] }} [grounding] + * @property {{ question: string, previousFields: string[], operation: { key: string, method: string, path: string, summary: string, description: string, parameters: { name: string }[], response: { fields?: { name: string, type: string, description?: string }[] } }, arguments: Record, requestPath: string, data: unknown, truncated: boolean }} [apiGrounding] + * + * @typedef {Object} AskAnswer + * @property {string} output + * @property {import("../storage.js").StoredArtifact[]} artifacts + * @property {string[]} [metricPaths] + * @property {import("../storage.js").ApiContext} [apiContext] + * @property {import("../storage.js").SourceContext[]} [sourceContext] + * @property {import("../storage.js").KnowledgeContext} [knowledgeContext] + * @property {import("../storage.js").StoredChat} chat */ +/** + * @param {import("../model.js").AskModel} model + * @param {NonNullable} grounding + * @param {(status: string) => void} onStatus + */ +async function answerFromEvidence(model, grounding, onStatus) { + onStatus("Answering from source…"); + const result = await model.generate( + [ + { + role: "system", + content: "Answer the exact request in at most 45 words and normal sentence casing using only verified facts, metric metadata, and source excerpts. Evidence is strongest first; ignore later excerpts unless needed. A declaration proves its definition and literal return type; a call expression proves its caller. Copy provided metric names, code identifiers, and types exactly; never respell, expand, or abbreviate them. Answer directly, never discuss the request's wording. Do not add background knowledge or guesses.", + }, + { + role: "user", + content: JSON.stringify(grounding), + }, + ], + () => {}, + [], + "none", + { maxTokens: 72 }, + ); + const answer = result.text.trim(); + const sources = grounding.excerpts.slice(0, 2); + const fallback = !answer && sources[0] + ? `The strongest verified source match is \`${sources[0].path}:${sources[0].startLine}${sources[0].endLine ? `-${sources[0].endLine}` : ""}\`.` + : ""; + return { + output: renderEvidence({ + facts: [ + answer || fallback, + ...grounding.facts, + ].filter(Boolean), + sources, + excerpts: [], + }), + sourceContext: sources, + knowledgeContext: answer + ? { + title: grounding.metrics[0]?.name ?? grounding.question.slice(0, 160), + description: answer, + } + : undefined, + }; +} + /** * @param {import("../model.js").AskModel} model * @param {NonNullable} grounding @@ -95,137 +134,199 @@ function modelStatus(session) { */ async function answerFromApi(model, grounding, onStatus) { const apiAnswer = createApiAnswerTool(grounding); - const calculation = directApiCalculation( - grounding.question, - apiAnswer.fields, - grounding, + const question = ` ${normalize(grounding.question)} `; + const parameterNames = new Set( + grounding.operation.parameters.map(({ name }) => normalize(name)), ); - if (calculation) return calculation; - const direct = renderDirectApiAnswer(grounding); - if (direct) return direct; - + const mentionedResponse = ( + grounding.operation.response.fields ?? [] + ).some(({ name }) => { + const field = normalize(name.split(".").at(-1)); + return field && !parameterNames.has(field) && + question.includes(` ${field} `); + }); + const directFields = apiAnswer.fields.filter((field) => { + const name = normalize(field.name.split(".").at(-1)); + return name && question.includes(` ${name} `); + }); + const requestTokens = new Set( + normalize(grounding.question).split(" ").filter((token) => token.length > 2), + ); + const canSelectDirectly = (/** @type {typeof apiAnswer.fields[number]} */ field) => { + const ownTokens = new Set(normalize(field.name).split(" ")); + const qualifiers = [...requestTokens].filter((token) => !ownTokens.has(token)); + return !apiAnswer.fields.some((candidate) => + candidate.name !== field.name && + qualifiers.some((token) => + normalize(`${candidate.name} ${candidate.description ?? ""}`) + .split(" ") + .includes(token) + ) + ); + }; + if (directFields.length === 1 && canSelectDirectly(directFields[0])) { + const field = directFields[0]; + return { + output: finishApiAnswer( + "select_api_field", + { + field: field.ref, + label: field.name.split(".").at(-1)?.replaceAll("_", " "), + }, + apiAnswer.fields, + grounding, + ), + fields: [field.name], + }; + } + if (apiAnswer.resolved && canSelectDirectly(apiAnswer.resolved)) { + const field = apiAnswer.resolved; + return { + output: finishApiAnswer( + "select_api_field", + { field: field.ref }, + apiAnswer.fields, + grounding, + ), + fields: [field.name], + }; + } + if (apiAnswer.ambiguous.length > 1) { + const choices = apiAnswer.ambiguous + .map((field) => + `**${field.name.replaceAll(".", " · ").replaceAll("_", " ")}**${ + field.description ? ` — ${field.description}` : "" + }` + ) + .join("\n- "); + return { + output: finishApiAnswer( + "answer_api_text", + { + text: `I found multiple matching fields:\n- ${choices}\n\nWhich one do you mean?`, + }, + apiAnswer.fields, + grounding, + ), + fields: apiAnswer.ambiguous.map(({ name }) => name), + }; + } + if ( + !grounding.previousFields?.length && + !mentionedResponse && + !apiAnswer.direct + ) { + return summarizeApiAnswer(grounding); + } + if (apiAnswer.direct) { + const field = apiAnswer.direct; + return { + output: finishApiAnswer( + "select_api_field", + { field: field.ref }, + apiAnswer.fields, + grounding, + ), + fields: [field.name], + }; + } onStatus("Answering from API…"); const instruction = apiAnswer.fields.length - ? "Answer the user's exact question using only the verified API result and schema. Call answer_from_api once. Choose calculate whenever the requested numeric value combines fields. Use operator with ordered operands for ordinary arithmetic, or terms for a signed sum. Choose answer only when no arithmetic is required. Preserve identifiers and units. Never invent missing values." - : "Answer the user's exact question using only the verified API result and schema. Call answer_from_api once with answer. Preserve identifiers and units. Never invent missing values."; - const answer = await model.generate( - [ - { - role: "system", - content: instruction, - }, - { - role: "user", - content: JSON.stringify(grounding), - }, - ], - () => {}, - [apiAnswer.tool], - { name: "answer_from_api" }, - { maxTokens: 128 }, - ); - const call = answer.toolCalls[0]; - if (!call || call.name !== "answer_from_api") { - throw new Error("The AI did not produce a valid API answer"); + ? `Answer the exact newest request using only the verified API result. Call exactly one matching tool. Select a raw field only when that field itself was requested${apiAnswer.previous ? "; continue the preceding numeric answer when the request applies arithmetic to it" : ""}. When the requested concept is narrower than an aggregate field, derive it from matching component fields. Never replace requested arithmetic with a convenient field. For subtraction and division, keep operands in the request's arithmetic order: minuend or dividend first. Preserve identifiers and units. Never invent missing values.` + : "Answer the exact request using only the verified API result. Call answer_api_text exactly once. Preserve identifiers and units. Never invent missing values."; + const prompt = { + question: grounding.question, + previous: apiAnswer.previous + ? { + ref: apiAnswer.previous.ref, + name: apiAnswer.previous.name, + type: apiAnswer.previous.type, + value: apiAnswer.previous.value, + } + : undefined, + fields: apiAnswer.fields.map(({ ref, name, type, description, value }) => ({ + ref, + name, + type, + description, + value, + })), + ...(!apiAnswer.previous ? { data: grounding.data } : {}), + }; + const generateAnswer = (extra = "") => + model.generate( + [ + { + role: "system", + content: extra ? `${instruction} ${extra}` : instruction, + }, + { role: "user", content: JSON.stringify(prompt) }, + ], + () => {}, + apiAnswer.tools, + { name: "answer_api" }, + { maxTokens: 72 }, + ); + let answer = await generateAnswer(); + let call = answer.toolCalls[0]; + if (!call || call.name !== "answer_api") { + return summarizeApiAnswer(grounding); } - return finishApiAnswer(call.arguments, apiAnswer.fields, grounding); -} - -/** @param {NonNullable[]} groundings */ -function answerFromApiComparison(groundings) { - const rows = groundings.map((grounding) => { - const answer = renderDirectApiAnswer(grounding); - if (!answer) return undefined; - const label = Object.values(grounding.arguments).join(", ") || grounding.requestPath; - return `**${label}**\n${answer.replace(/\n\nData:.*$/s, "")}`; - }); - if (rows.some((row) => !row)) { - throw new Error("The API comparison was ambiguous"); + const actionFor = (/** @type {Record} */ arguments_) => + arguments_.action === "select" + ? "select_api_field" + : arguments_.action === "continue" + ? "continue_api_calculation" + : arguments_.action === "calculate" + ? "calculate_api_fields" + : arguments_.action === "text" + ? "answer_api_text" + : ""; + let actionName = actionFor(call.arguments); + const selectedField = actionName === "select_api_field" + ? apiAnswer.fields.find(({ ref }) => ref === call.arguments.field) + : undefined; + if (selectedField && !canSelectDirectly(selectedField)) { + answer = await generateAnswer( + `Do not select ${selectedField.ref} (${selectedField.name}): its schema scope does not satisfy all request qualifiers. Derive the requested result from matching component fields or choose an exact narrower field.`, + ); + call = answer.toolCalls[0]; + if (!call || call.name !== "answer_api") { + return summarizeApiAnswer(grounding); + } + actionName = actionFor(call.arguments); } - const operation = groundings[0].operation; - return `${rows.join("\n\n")}\n\nData: \`${operation.method} ${operation.path}\``; -} - -/** - * @param {import("../model.js").AskModel} model - * @param {NonNullable} grounding - * @param {(update: import("../model.js").TokenUpdate) => void} onToken - * @param {(status: string) => void} onStatus - */ -async function answerFromSource(model, grounding, onToken, onStatus) { - const direct = directSourceFact(grounding.question, grounding.excerpts); - if (direct) { - return renderEvidence({ - facts: [direct], - sources: grounding.excerpts, - excerpts: [], - }); + if (!actionName) return summarizeApiAnswer(grounding); + const selectedRefs = actionName === "select_api_field" + ? [call.arguments.field] + : actionName === "continue_api_calculation" + ? [apiAnswer.previous?.ref, call.arguments.operand] + : typeof call.arguments.left === "string" && + typeof call.arguments.right === "string" + ? [call.arguments.left, call.arguments.right] + : Array.isArray(call.arguments.operands) + ? call.arguments.operands + : []; + const selected = new Set(selectedRefs.map(String)); + try { + return { + output: finishApiAnswer( + actionName, + call.arguments, + apiAnswer.fields, + grounding, + ), + fields: apiAnswer.fields + .filter(({ ref }) => selected.has(ref)) + .map(({ name }) => name), + }; + } catch { + return summarizeApiAnswer(grounding); } - onStatus("Answering from source…"); - const instruction = grounding.metric - ? "Explain only the verified metric in plain language in at most 45 words. Start with its exact metric name in normal words and state its verified unit. Never describe the metric as denominated in another unit; source quantities may use other units only in its calculation. Use the source excerpt only for its computation; ignore sibling metrics. Preserve every comparison direction and arithmetic operation literally. Never change unrealized into realized. Do not add unsupported details." - : "Answer in at most 45 words using only the supplied source excerpt. Describe operations in source order. Preserve the exact subject, object, and identifiers of each relationship; never merge separate statements. Do not add background knowledge, guesses, or uncited details."; - const answer = await model.generate( - [ - { - role: "system", - content: instruction, - }, - { - role: "user", - content: JSON.stringify(grounding), - }, - ], - onToken, - [], - "none", - { maxTokens: 64 }, - ); - const text = answer.text.trim(); - if (!text || /\?\s*$/.test(text)) { - return renderEvidence({ - facts: [], - sources: [], - excerpts: grounding.excerpts, - }); - } - return renderEvidence({ - facts: [text], - sources: grounding.excerpts, - excerpts: [], - }); -} - -/** - * @param {import("../model.js").AskModel} model - * @param {NonNullable} grounding - * @param {(update: import("../model.js").TokenUpdate) => void} onToken - * @param {(status: string) => void} onStatus - */ -async function answerFromKnowledge(model, grounding, onToken, onStatus) { - onStatus("Answering from context…"); - const answer = await model.generate( - [ - { - role: "system", - content: "Answer the follow-up in at most 55 words using only the verified concept description. Keep the exact subject. An analogy may simplify the description but must preserve its meaning. Do not invent benefits, risks, mechanisms, or tradeoffs. If the description is insufficient, ask one short clarification question.", - }, - { - role: "user", - content: JSON.stringify(grounding), - }, - ], - onToken, - [], - "none", - { maxTokens: 80 }, - ); - return answer.text.trim(); } export function createAskTools() { const source = new AskSource(); - const contextTools = [searchTool()]; /** @type {AbortController | undefined} */ let controller; @@ -238,10 +339,6 @@ export function createAskTools() { ]); }, - toolsFor() { - return contextTools; - }, - /** * @param {Object} options * @param {string} options.question @@ -250,177 +347,234 @@ export function createAskTools() { * @param {() => Promise<{ chat: import("../storage.js").StoredChat, messages: import("../model.js").ChatMessage[] }>} options.prepare * @param {(update: import("../model.js").TokenUpdate) => void} options.onToken * @param {(status: string) => void} options.onStatus + * @returns {Promise} */ - async answer({ question, history, model, prepare, onToken, onStatus }) { + async answer({ + question, + history, + model, + prepare, + onToken: _onToken, + onStatus, + }) { controller = new AbortController(); const { signal } = controller; - const session = new AskToolSession(source); - await session.begin( - question, - history, - () => onStatus("Indexing tools…"), - ); try { - let prepared; - if (session.verifyDirectApiIntent) { - prepared = await prepare(); - onStatus("Understanding request…"); - const frontDoor = await useFrontDoor(model, prepared.messages); - if (frontDoor.kind === "answer") { - return { - output: frontDoor.text, - artifacts: [], - metricPaths: [], - chat: prepared.chat, - }; - } - } + const session = new AskToolSession(source); + const [prepared] = await Promise.all([ + prepare(), + session.begin(question, history, onStatus), + ]); + signal.throwIfAborted(); - const direct = /** @type {ToolOutcome | undefined} */ ( - await session.tryDirect(onStatus, signal) - ); - if (direct?.apiGrounding) { - return { - output: await answerFromApi(model, direct.apiGrounding, onStatus), - artifacts: [], - metricPaths: session.metricPaths(), - apiContext: session.apiContext(), - }; - } - if (direct?.apiGroundings) { - return { - output: answerFromApiComparison(direct.apiGroundings), - artifacts: [], - metricPaths: session.metricPaths(), - apiContext: session.apiContext(), - }; - } - if (direct?.grounding) { - return { - output: await answerFromSource( - model, - direct.grounding, - onToken, - onStatus, - ), - artifacts: [], - metricPaths: session.metricPaths(), - sourceContext: direct.grounding.excerpts, - }; - } - if (direct?.knowledgeGrounding) { - return { - output: await answerFromKnowledge( - model, - direct.knowledgeGrounding, - onToken, - onStatus, - ), - artifacts: [], - metricPaths: session.metricPaths(), - knowledgeContext: direct.knowledgeGrounding.context, - }; - } - if (direct) return { ...direct, metricPaths: session.metricPaths() }; - - prepared ??= await prepare(); - const { messages } = prepared; - if (!session.requiresTools) { - onStatus("Understanding request…"); - const frontDoor = await useFrontDoor(model, messages); - if (frontDoor.kind === "answer") { - return { - output: frontDoor.text, - artifacts: [], - metricPaths: [], - chat: prepared.chat, - }; - } - } - - for (let round = 0; round < MAX_TOOL_ROUNDS; round += 1) { - signal.throwIfAborted(); - onStatus(modelStatus(session)); - const newestUser = messages.findLast((message) => message.role === "user"); - const stagedMessages = [ - { role: /** @type {const} */ ("system"), content: session.instruction() }, - ...(newestUser ? [newestUser] : []), - ...(session.observation - ? [{ - role: /** @type {const} */ ("user"), - content: `Available source-derived result:\n${JSON.stringify(session.observation)}`, - }] - : []), - ]; - const result = await model.generate( - stagedMessages, + /** @type {{ action: string, call?: import("../model.js").ToolCall } | undefined} */ + const direct = session.directRoute(); + let call = direct?.call; + let action = direct?.action ?? ""; + if (!action) { + onStatus("Choosing capability…"); + const routeTools = session.routeTools(); + const route = await model.generate( + session.routeMessages(), () => {}, - [await session.tool()], - { name: "next_action" }, - { maxTokens: 64 }, + routeTools, + { name: "choose_capability" }, + { maxTokens: 48 }, ); - const call = result.toolCalls[0]; - if (!call || call.name !== "next_action") { - throw new Error("The AI did not choose a valid action"); + const selected = route.toolCalls[0]; + const sourceQuery = selected?.name === "choose_capability" && + typeof selected.arguments.sourceQuery === "string" + ? selected.arguments.sourceQuery + : ""; + const selectedCapability = selected?.name === "choose_capability" && + typeof selected.arguments.capability === "string" + ? selected.arguments.capability + : ""; + action = sourceQuery && + ( + ( + selectedCapability === "answer_general" && + session.hasSourceContext() + ) || + selectedCapability === "search_source" + ) + ? "search_source" + : selectedCapability; + if (!action) { + throw new Error("The AI did not choose a valid capability"); } + call = action === "call_api" && + typeof selected?.arguments.apiRef === "string" + ? { + name: action, + arguments: { ref: selected.arguments.apiRef }, + } + : action === "search_source" + ? { + name: action, + arguments: { + query: sourceQuery || question, + }, + } + : session.directCall(action); + } - signal.throwIfAborted(); - const outcome = /** @type {ToolOutcome} */ ( - await session.execute(call.arguments, onStatus, signal) - ); - if (!outcome.done) continue; - if (outcome.general) { - onStatus("Answering…"); - const answer = await model.generate( + signal.throwIfAborted(); + await session.prepareAction(action, onStatus); + signal.throwIfAborted(); + if (!call || action === "explain_evidence") { + call = session.directCall(action) ?? call; + } + if (!call) { + onStatus("Understanding request…"); + if (action === "answer_general") { + const messages = session.actionMessages(action); + let result = await model.generate( messages, - onToken, + () => {}, [], "none", + { maxTokens: 96 }, ); + if (unsupportedNumbers(result.text, messages).size) { + result = await model.generate( + [ + ...messages, + { + role: "assistant", + content: result.text, + }, + { + role: "user", + content: "Replace the draft with a direct answer containing no unsupported quantities. Keep established static Bitcoin facts only when the request directly needs them; otherwise use qualitative examples. Return only the replacement answer. Never mention the draft, review, evidence, context, or these instructions.", + }, + ], + () => {}, + [], + "none", + { maxTokens: 96 }, + ); + } + const answer = removeUnsupportedQuantitySentences( + result.text, + messages, + ) || "I do not have enough verified context to answer that without guessing."; + call = { + name: action, + arguments: { answer }, + }; + } else { + const result = await model.generate( + session.actionMessages(action), + () => {}, + [session.actionTool(action)], + { name: action }, + { maxTokens: action === "call_api" ? 128 : 64 }, + ); + call = result.toolCalls[0]; + } + } + if (!call || call.name !== action) { + throw new Error("The AI did not complete the selected capability"); + } + if (action === "call_api") { + const ref = typeof call.arguments.ref === "string" + ? call.arguments.ref + : ""; + if (!ref) throw new Error("The AI did not select an API operation"); + let arguments_ = session.apiArguments(ref); + if (!session.hasApiArguments(ref, arguments_)) { + onStatus("Reading API arguments…"); + const argumentsResult = await model.generate( + session.apiArgumentMessages(ref), + () => {}, + [session.apiArgumentTool(ref)], + { name: "provide_api_arguments" }, + { maxTokens: 96 }, + ); + const argumentsCall = argumentsResult.toolCalls[0]; + if ( + !argumentsCall || + argumentsCall.name !== "provide_api_arguments" + ) { + throw new Error("The AI did not provide valid API arguments"); + } + arguments_ = { + ...(argumentsCall.arguments.arguments ?? {}), + ...arguments_, + }; + } + arguments_ = session.validateApiArguments(ref, arguments_); + const missing = session.missingApiArguments(ref, arguments_); + if (missing.length) { + const subject = missing + .map((/** @type {any} */ parameter) => + parameter.description || parameter.name + ) + .join(" and "); return { - output: answer.text, + output: `Which ${subject} should I use?`, artifacts: [], - metricPaths: [], - chat: prepared.chat, - }; - } - if (outcome.grounding) { - return { - output: await answerFromSource( - model, - outcome.grounding, - onToken, - onStatus, - ), - artifacts: [], - metricPaths: session.metricPaths(), - sourceContext: outcome.grounding.excerpts, - chat: prepared.chat, - }; - } - if (outcome.apiGrounding) { - return { - output: await answerFromApi( - model, - outcome.apiGrounding, - onStatus, - ), - artifacts: [], - metricPaths: session.metricPaths(), - apiContext: session.apiContext(), chat: prepared.chat, }; } + call = { + name: action, + arguments: { + ref, + arguments: arguments_, + }, + }; + } + + const outcome = /** @type {ToolOutcome} */ ( + await session.execute(call, onStatus, signal) + ); + if (outcome.apiGrounding) { + const answered = await answerFromApi( + model, + outcome.apiGrounding, + onStatus, + ); return { - output: outcome.output ?? "", - artifacts: outcome.artifacts ?? [], - metricPaths: session.metricPaths(), - apiContext: session.apiContext(), + output: answered.output, + artifacts: [], + apiContext: outcome.apiContext + ? { + ...outcome.apiContext, + ...(answered.fields.length + ? { fields: answered.fields } + : {}), + } + : undefined, chat: prepared.chat, }; } - throw new Error("The AI used too many tool steps. Try a more specific question."); + if (outcome.grounding) { + const grounded = await answerFromEvidence( + model, + outcome.grounding, + onStatus, + ); + return { + output: grounded.output, + artifacts: [], + metricPaths: outcome.metricPaths, + sourceContext: grounded.sourceContext, + knowledgeContext: grounded.knowledgeContext, + chat: prepared.chat, + }; + } + return { + output: outcome.output ?? "", + artifacts: outcome.artifacts ?? [], + metricPaths: outcome.metricPaths, + apiContext: outcome.apiContext, + sourceContext: outcome.sourceContext, + knowledgeContext: outcome.knowledgeContext, + chat: prepared.chat, + }; } finally { controller = undefined; onStatus(""); diff --git a/website_next/ask/tools/metrics/index.js b/website_next/ask/tools/metrics/index.js index d73cf425c..840fd7c0c 100644 --- a/website_next/ask/tools/metrics/index.js +++ b/website_next/ask/tools/metrics/index.js @@ -1,12 +1,11 @@ import { BRK_BASE_URL, brk } from "../../../utils/client.js"; import { WorkerClient } from "../worker-client.js"; -import { canonicalMetricQuery, expandMetricQueries } from "./language.js"; const WORKER_URL = import.meta.resolve("./worker.js"); const SERIES_URL = `${BRK_BASE_URL}/api/series`; const FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]); -/** @typedef {{ path: string, name: string, endpoint: string, indexes: string[], type: string, suggestedUnit?: string, matchedQuery?: string, score?: number }} CatalogMetric */ +/** @typedef {{ path: string, name: string, endpoint: string, indexes: string[], type: string, suggestedUnit?: string, matchedQuery?: string, matchedTerms?: number, specificity?: number, relevance?: number, score?: number }} CatalogMetric */ /** @param {unknown} value */ function isMetric(value) { @@ -74,14 +73,14 @@ export function prewarmMetricIndex() { export function searchMetrics(queries, limit = 16, prefixes = [], onProgress) { return index.request( "search", - { queries: expandMetricQueries(queries), limit, prefixes }, + { queries, limit, prefixes }, onProgress, ); } -/** @param {string} query @param {(() => void) | undefined} [onProgress] @returns {Promise} */ -export function mentionedMetrics(query, onProgress) { - return index.request("mentions", { query: canonicalMetricQuery(query) }, onProgress); +/** @param {string} query @param {(() => void) | undefined} [onProgress] @returns {Promise} */ +export function mentionedMetricNames(query, onProgress) { + return index.request("mentions", { query }, onProgress); } /** @param {string} name @returns {Promise} */ @@ -94,7 +93,7 @@ export function metricsByPaths(paths, onProgress) { return index.request("byPaths", { paths }, onProgress); } -/** @param {{ name: string }} metric @param {string} query @returns {Promise<{ totalSeries: number, groups: { family: string, examples: string[] }[], series: CatalogMetric[] } | undefined>} */ +/** @param {{ name: string }} metric @param {string} query @returns {Promise<{ totalSeries: number, groups: { family: string, examples: string[] }[], series: (CatalogMetric & { selector: string, matchedTerms: number })[] } | undefined>} */ export function metricVariants(metric, query = "") { return index.request("variants", { name: metric.name, diff --git a/website_next/ask/tools/metrics/language.js b/website_next/ask/tools/metrics/language.js deleted file mode 100644 index 6192870dd..000000000 --- a/website_next/ask/tools/metrics/language.js +++ /dev/null @@ -1,41 +0,0 @@ -/** @type {[RegExp, string][]} */ -const ALIASES = [ - [/\bcapitalised\b/g, "capitalized"], - [/\bcap price\b/g, "capitalized price"], - [/\ball time high\b/g, "ath"], - [/\blong term holders?\b/g, "lth"], - [/\bshort term holders?\b/g, "sth"], - [/\blong term\b/g, "lth"], - [/\bshort term\b/g, "sth"], - [/\b(lth|sth)\s+holders?\b/g, "$1"], - [/\btransactions?\s+count\b/g, "tx count"], - [/\b(?:one )?(?:bitcoin|btc) worth\b/g, "bitcoin spot price"], -]; - -/** @param {string} query */ -function expand(query) { - return ALIASES.reduce( - (value, [pattern, replacement]) => value.replace(pattern, replacement), - query.toLowerCase().replace(/[-_]+/g, " "), - ) - .replace(/\b(?:over|through)\s+time\b|\btime\s+series\b/g, " ") - .replace( - /\b(\d+(?:\.\d+)?)\s+to\s+(\d+(?:\.\d+)?)\s*(btc|sats?)\b/g, - "$1$3 to $2$3", - ) - .replace(/\s+/g, " ") - .trim(); -} - -/** @param {string} query */ -export function canonicalMetricQuery(query) { - return expand(query); -} - -/** @param {string[]} queries */ -export function expandMetricQueries(queries) { - return [...new Set(queries.flatMap((query) => { - const expanded = expand(query); - return expanded === query ? [query] : [expanded, query]; - }))]; -} diff --git a/website_next/ask/tools/metrics/options.js b/website_next/ask/tools/metrics/options.js deleted file mode 100644 index 991745834..000000000 --- a/website_next/ask/tools/metrics/options.js +++ /dev/null @@ -1,109 +0,0 @@ -import { normalize } from "../text.js"; -import { searchMetrics } from "./index.js"; -import { canonicalMetricQuery } from "./language.js"; - -export const MAX_OPTIONS = 12; - -/** @param {string[]} topics @param {() => void} onProgress */ -export async function exactTopicMetrics(topics, onProgress) { - if (!topics.length) return []; - - const names = new Set(topics.map(normalize)); - const metrics = await searchMetrics(topics, MAX_OPTIONS, [], onProgress); - return metrics.filter((metric) => names.has(normalize(metric.name))); -} - -/** - * Resolve coordinated metric wording only when expanding its shared suffix - * produces two exact generated catalog names. - * @param {string} request - * @param {() => void} onProgress - */ -export async function coordinatedMetrics(request, onProgress) { - const expression = canonicalMetricQuery(request) - .replace( - /^(?:(?:compare|chart|graph|plot|show|visualize|visualise)(?: me)?|comparison of)\s+/, - "", - ) - .replace(/^both\s+/, "") - .trim(); - const parts = expression - .split(/\s+(?:and|against|versus|vs)\s+/) - .map((part) => part.trim()) - .filter(Boolean); - if (parts.length !== 2) return []; - - const [left, right] = parts; - const candidates = [[left, right]]; - const leftWords = left.split(" "); - const rightWords = right.split(" "); - for (let index = 1; index < rightWords.length; index += 1) { - candidates.push([`${left} ${rightWords.slice(index).join(" ")}`, right]); - } - for (let index = 1; index < leftWords.length; index += 1) { - candidates.push([left, `${right} ${leftWords.slice(index).join(" ")}`]); - } - - for (const topics of candidates) { - const metrics = await exactTopicMetrics(topics, onProgress); - if (new Set(metrics.map((metric) => metric.path)).size === 2) return metrics; - } - return []; -} - -/** @param {any[]} items */ -export function uniqueMetricOptions(items) { - return [...new Map(items.map((/** @type {any} */ item) => [item.ref, item])).values()]; -} - -/** @param {any[][]} groups */ -export function mergeMetricGroups(groups) { - const output = []; - const positions = new Map(); - const ranks = Math.max(...groups.map((group) => group.length), 0); - - for (let rank = 0; rank < ranks && output.length < MAX_OPTIONS; rank += 1) { - for (const group of groups) { - const metric = group[rank]; - if (!metric) continue; - - const position = positions.get(metric.path); - if (position !== undefined) { - const current = output[position]; - const exact = normalize(metric.name) === normalize(metric.matchedQuery ?? ""); - const currentExact = normalize(current.name) === normalize(current.matchedQuery ?? ""); - if (exact && !currentExact) output[position] = metric; - continue; - } - positions.set(metric.path, output.length); - output.push(metric); - if (output.length === MAX_OPTIONS) break; - } - } - return output; -} - -/** @param {any[]} items */ -export function balancedOptions(items) { - const order = ["fact", "guide", "metric", "source"]; - const groups = order - .map((kind) => items - .filter((item) => item.kind === kind) - .sort((left, right) => right.score - left.score)) - .filter((group) => group.length); - const output = []; - - for (let rank = 0; output.length < MAX_OPTIONS; rank += 1) { - let added = false; - for (const group of groups) { - const item = group[rank]; - if (!item) continue; - const { score, ...option } = item; - output.push(option); - added = true; - if (output.length === MAX_OPTIONS) break; - } - if (!added) break; - } - return output; -} diff --git a/website_next/ask/tools/metrics/unit.js b/website_next/ask/tools/metrics/unit.js new file mode 100644 index 000000000..4c004a10d --- /dev/null +++ b/website_next/ask/tools/metrics/unit.js @@ -0,0 +1,11 @@ +/** @param {string} type */ +export function unitFromType(type) { + const value = type.toLowerCase(); + if (value.includes("dollar") || value.includes("usd") || value.includes("cents")) return "usd"; + if (value.includes("bitcoin") || value === "btc") return "btc"; + if (value.includes("percent")) return "percent"; + if (value.includes("address")) return "addresses"; + if (value.includes("utxo") || value.includes("output")) return "utxos"; + if (value.includes("block") || value.includes("height")) return "blocks"; + return "number"; +} diff --git a/website_next/ask/tools/metrics/worker.js b/website_next/ask/tools/metrics/worker.js index 351e8c580..e5d61e2c7 100644 --- a/website_next/ask/tools/metrics/worker.js +++ b/website_next/ask/tools/metrics/worker.js @@ -1,9 +1,9 @@ import { QuickMatch, QuickMatchConfig } from "../../../modules/quickmatch-js/0.5.0/src/index.js"; -import { normalize, tokenAffinity } from "../text.js"; +import { normalize, relevance } from "../text.js"; import { metricsFromSeries } from "./series.js"; +import { unitFromType } from "./unit.js"; const SEARCH_CANDIDATES = 1_024; -const MAX_MENTION_WORDS = 12; /** @typedef {{ path: string, name: string, indexes: string[], type: string, document: string }} CatalogMetric */ @@ -12,21 +12,22 @@ function searchable(value) { return normalize(value); } -/** @param {string} path @param {string} type */ -function suggestedUnit(path, type) { - if (/(dollar|usd|cents)/i.test(type)) return "usd"; - if (/(bitcoin|btc)/i.test(type)) return "btc"; - if (/(percent|ratio)/i.test(type)) return "percent"; - if (/address/i.test(type)) return "addresses"; - if (/(utxo|output)/i.test(type)) return "utxos"; - if (/(block|height)/i.test(type)) return "blocks"; - if (/(percent|ratio|dominance)/i.test(path)) return "percent"; - if (/(usd|price|cap)/i.test(path)) return "usd"; - if (/(btc|supply|value)/i.test(path)) return "btc"; - if (/(address|addr)/i.test(path)) return "addresses"; - if (/(?:^|\.)(?:outputs?|unspentCount|spentCount)(?:\.|$)/i.test(path)) return "utxos"; - if (/(block|height|epoch)/i.test(path)) return "blocks"; - return "number"; +/** @param {string} value */ +function queryVocabulary(value) { + const words = searchable(value).split(" ").filter(Boolean); + const vocabulary = new Set(words); + for (let start = 0; start < words.length; start += 1) { + for ( + let length = 2; + length <= 5 && start + length <= words.length; + length += 1 + ) { + vocabulary.add( + words.slice(start, start + length).map((word) => word[0]).join(""), + ); + } + } + return vocabulary; } /** @param {number} limit */ @@ -51,44 +52,57 @@ async function buildState(url) { const byName = new Map(); /** @type {Map} */ const byPath = new Map(); - /** @type {Map} */ - const bySearchableName = new Map(); + const metricNames = [...new Set(items.map((metric) => searchable(metric.name)))]; /** @type {Map} */ const byDocument = new Map(); + const documentFrequency = new Map(); for (const metric of items) { if (!byName.has(metric.name)) byName.set(metric.name, metric); + const normalizedName = searchable(metric.name); + if (!byName.has(normalizedName)) byName.set(normalizedName, metric); byPath.set(metric.path, metric); - const nameKey = searchable(metric.name); - const named = bySearchableName.get(nameKey) ?? []; - named.push(metric); - bySearchableName.set(nameKey, named); if (!byDocument.has(metric.document)) byDocument.set(metric.document, metric); + for (const token of new Set(metric.document.split(" ").filter(Boolean))) { + documentFrequency.set( + token, + (documentFrequency.get(token) ?? 0) + 1, + ); + } } const config = createConfig(); const matcher = new QuickMatch(items.map(({ document }) => document), config); - const nameConfig = createConfig(12).withTrigramBudget(4); - const nameMatcher = new QuickMatch([...bySearchableName.keys()], nameConfig); - const nameWords = new Set( - [...bySearchableName.keys()].flatMap((name) => name.split(" ")), - ); /** @type {Map} */ const scoped = new Map(); return { items, byName, byPath, - bySearchableName, + metricNames, byDocument, + documentFrequency, matcher, config, - nameMatcher, - nameConfig, - nameWords, scoped, }; } +/** @param {Awaited>} index @param {string} query */ +function mentions(index, query) { + const value = ` ${searchable(query)} `; + const names = index.metricNames + .filter((name) => name && value.includes(` ${name} `)) + .filter((name, _, matches) => + !matches.some((candidate) => + candidate !== name && + candidate.length > name.length && + ` ${candidate} `.includes(` ${name} `) + ) + ) + .sort((left, right) => right.length - left.length || left.localeCompare(right)); + return names.slice(0, 4); +} + /** @type {Promise>> | undefined} */ let statePromise; let stateUrl = ""; @@ -110,7 +124,7 @@ function publicMetric(metric) { name: metric.name, indexes: metric.indexes, type: metric.type, - suggestedUnit: suggestedUnit(metric.path, metric.type), + suggestedUnit: unitFromType(metric.type), }; } @@ -172,11 +186,27 @@ function searchOne(index, query, limit, prefixes) { .map((document) => index.byDocument.get(document)) .filter((metric) => metric && scope.inScope(metric)) .slice(0, limit) - .map((metric, rank) => ({ - ...publicMetric(/** @type {CatalogMetric} */ (metric)), - matchedQuery: query, - score: 1_000 - rank, - })); + .map((metric, rank) => { + const value = /** @type {CatalogMetric} */ (metric); + const documentTokens = new Set(value.document.split(" ")); + const queryTokens = [...new Set(normalizedQuery.split(" ").filter(Boolean))]; + const matchedTokens = queryTokens.filter((token) => + documentTokens.has(token) + ); + return { + ...publicMetric(value), + matchedQuery: query, + matchedTerms: matchedTokens.length, + specificity: matchedTokens.reduce((sum, token) => { + const frequency = index.documentFrequency.get(token) ?? + index.items.length; + return sum + + Math.log((index.items.length + 1) / (frequency + 1)) + 1; + }, 0), + relevance: relevance(query, value.document), + score: 1_000 - rank, + }; + }); } /** @param {Awaited>} index @param {string[]} queries @param {number} limit @param {string[]} prefixes */ @@ -200,78 +230,6 @@ function search(index, queries, limit, prefixes) { return output; } -/** @param {Awaited>} index @param {string} query */ -function mentions(index, query) { - const words = searchable(query).match(/[a-z0-9]+/g) ?? []; - /** @type {{ start: number, end: number, metric: CatalogMetric }[]} */ - const matches = []; - - for (let start = 0; start < words.length; start += 1) { - for ( - let end = start + 1; - end <= Math.min(words.length, start + MAX_MENTION_WORDS); - end += 1 - ) { - const phraseWords = words.slice(start, end); - const phrase = phraseWords.join(" "); - const named = index.bySearchableName.get(phrase); - if (named?.length === 1) { - matches.push({ start, end, metric: named[0] }); - continue; - } - - if ( - phraseWords.every((word) => index.nameWords.has(word)) || - phraseWords.length === 1 && phrase.length < 5 - ) continue; - const fuzzy = index.nameMatcher.matchesWith(phrase, index.nameConfig) - .map((name) => ({ - name, - named: index.bySearchableName.get(name), - candidateWords: name.split(" "), - })) - .filter(({ named, candidateWords }) => - named?.length === 1 && candidateWords.length === phraseWords.length - ) - .map((candidate) => { - const affinities = phraseWords.map((word, index_) => - tokenAffinity(word, candidate.candidateWords[index_]) - ); - return { - ...candidate, - affinities, - score: affinities.reduce((sum, affinity) => sum + affinity, 0) / - affinities.length, - }; - }) - .filter(({ affinities, score }) => - score >= 0.82 && affinities.every((affinity) => affinity >= 0.65) - ) - .sort((left, right) => right.score - left.score)[0]; - if (fuzzy) { - const [metric] = fuzzy.named ?? []; - if (!metric) continue; - matches.push({ - start, - end, - metric, - }); - } - } - } - - const maximal = matches.filter((match) => - !matches.some((candidate) => - candidate.start <= match.start && - candidate.end >= match.end && - candidate.end - candidate.start > match.end - match.start - ) - ); - return [...new Map( - maximal.map(({ metric }) => [metric.path, publicMetric(metric)]), - ).values()]; -} - /** * @param {Awaited>} index * @param {string} name @@ -304,12 +262,17 @@ function variants(index, name, path, query) { index.matcher.matchesWith(searchable(query), index.config.withLimit(SEARCH_CANDIDATES)) .map((document, rank) => [index.byDocument.get(document)?.path, rank]), ); + const queryTerms = queryVocabulary(query); const ranked = candidates .map((candidate) => ({ ...publicMetric(candidate), rank: preferredPaths.get(candidate.path) ?? SEARCH_CANDIDATES, + queryMatches: searchable(candidate.path) + .split(" ") + .filter((token) => queryTerms.has(token)).length, })) .sort((left, right) => + right.queryMatches - left.queryMatches || Number(right.name === name) - Number(left.name === name) || left.rank - right.rank || left.path.localeCompare(right.path) @@ -327,13 +290,26 @@ function variants(index, name, path, query) { commonSuffix = count ? commonSuffix.slice(-count) : []; } + const selectors = ranked.map((candidate) => { + const path = candidate.path.split("."); + return commonSuffix.length ? path.slice(0, -commonSuffix.length) : path; + }); + let commonPrefix = selectors[0] ?? []; + for (const selector of selectors.slice(1)) { + let count = 0; + while ( + count < commonPrefix.length && + count < selector.length && + commonPrefix[count] === selector[count] + ) count += 1; + commonPrefix = commonPrefix.slice(0, count); + } + const groups = new Map(); - for (const candidate of ranked) { - const selector = candidate.path.split(".").slice(0, -commonSuffix.length); - const cohortIndex = selector.indexOf("cohorts"); - const cohort = selector.slice(cohortIndex < 0 ? 0 : cohortIndex + 1); - const family = cohort.length > 2 ? cohort.slice(0, 2).join(" / ") : cohort[0] ?? "root"; - const value = cohort.length > 2 ? cohort.slice(2).join(" / ") : cohort[1] ?? "all"; + for (const selector of selectors) { + const varying = selector.slice(commonPrefix.length); + const family = varying[0] ?? commonPrefix.at(-1) ?? "root"; + const value = varying.slice(1).join(" / ") || varying[0] || "all"; const group = groups.get(family) ?? { family, count: 0, examples: [] }; group.count += 1; if (group.examples.length < 5) group.examples.push(value); @@ -343,11 +319,26 @@ function variants(index, name, path, query) { return { totalSeries: ranked.length, groups: [...groups.values()].slice(0, 8), - series: ranked.slice(0, 16).map(({ path, name: metricName, suggestedUnit }) => ({ - path, - name: metricName, - suggestedUnit, - })), + series: ranked.slice(0, 16).map(( + { path, name: metricName, suggestedUnit, indexes, type }, + index, + ) => { + const selector = selectors[index] + .slice(commonPrefix.length) + .join(" ") || selectors[index].at(-1) || ""; + const selectorTokens = new Set(searchable(selector).split(" ")); + return { + path, + name: metricName, + suggestedUnit, + indexes, + type, + selector, + matchedTerms: [...selectorTokens].filter((token) => + token && queryTerms.has(token) + ).length, + }; + }), }; } diff --git a/website_next/ask/tools/prompts.js b/website_next/ask/tools/prompts.js deleted file mode 100644 index a3afb74eb..000000000 --- a/website_next/ask/tools/prompts.js +++ /dev/null @@ -1,57 +0,0 @@ -const COMMON = `You route one step for Bitview's small on-device Bitcoin assistant. -Call next_action exactly once. Never answer directly, invent refs, alter returned refs, or repeat a ref. -Choose clarify when none of the available references match the user's meaning. Similar spelling alone is not a semantic match. Ask one short question; never force an unrelated result.`; - -export const ASK_STAGE_PROMPTS = /** @type {const} */ ({ - search: `You route one user request for Bitview's Bitcoin data assistant. -First decide whether the request needs Bitview/BRK evidence or tools at all. -Choose answer_general for ordinary Bitcoin knowledge, explanation, conversation, or writing that the model can answer without current site data or repository evidence. A related metric existing does not by itself make the request a metric lookup. -Choose clarify_request when the request depends on a missing subject or missing prior context. Ask for that subject; never search source merely to guess it. -Only return catalog queries for outcomes that actually need metric, API, source, data, or chart tools. Omit queries, context, and cardinality for answer_general and clarify_request. -Examples: -- With no previous topic, "Which holder group?" is clarify_request. -- After a verified capitalized price answer, "Which holder groups have it?" reuses the previous topic and explains verified variants. -- "Why does Bitcoin have a fixed supply?" is answer_general. -- "Tell me something about on-chain data" is answer_general. -- "Write a haiku about Bitcoin" is answer_general. -Choose the requested outcome and translate the user's meaning into terse catalog-style Bitcoin or BRK metric names or technical noun phrases. Never copy a question or include request verbs, pronouns, time words, or punctuation in a query. -Use one query for one metric. For X vs Y, return separate complete X and Y metric phrases; never leave vs or both sides inside one query. -Set cardinality to multiple for every comparison or request involving more than one distinct metric, even if you accidentally return one query. -Choose reuse_previous when the newest request asks another question about the previous verified topic, including its variants, cohorts, source, value, or chart. Choose extend_previous only when it adds a distinct new metric. Do not turn properties of the previous answer into new metric queries. -Choose read_requested_value for a current or historical number. -Choose read_api for a concrete blockchain record or server resource that should be read from Bitview's API, such as a transaction, address, block, mempool, fee estimate, or server status. Do not choose it for time-series metrics or ordinary Bitcoin knowledge. -Choose build_requested_chart for a graph, trend, history, comparison over time, or a request to show quantitative metrics over time—even when the user does not say chart. -When an active chart is supplied, choose edit_existing_chart only to add, remove, or replace series on that chart. -Choose explain_from_verified_facts for what or why questions, meaning, availability, cohorts, variants, or source code. -Choose answer_general when the request needs no repository evidence, live data, metric lookup, or chart. -Choose clarify_request when essential context is absent or multiple materially different interpretations remain and choosing one would change the result. Put one concise question in clarification. Never clarify merely because wording is informal. -Interpret ordinary wording by meaning. BRK means the software repository, not a coin. -Call next_action exactly once.`, - - explain: `${COMMON} -Choose the smallest sufficient set of returned references by semantic fit and call answer. Prefer recommended references when they answer the request.`, - - rewrite: `${COMMON} -Rewrite the newest request as concise conventional Bitcoin or BRK metric or source-search phrases. Translate colloquial meaning into standard technical terminology. Keep independently requested metrics as separate queries. -Return exactly one rewritten query for every supplied unmatched query, in the same order. Never merge comparison sides. -Return only those searches through next_action.`, - - read: `${COMMON} -Choose the smallest exact metric set that answers the requested value and call read_data. -Use latest for the present. Use at for a specific block or date, put that block or date in at, and choose height for a block.`, - - api: `${COMMON} -Choose the single read-only API operation that directly answers the request and call call_api. -Copy identifiers and parameter values exactly from the newest user request. Reuse previous verified arguments only for a dependent follow-up on the same resource. -If a required argument is absent, clarify instead of inventing it.`, - - chart: `${COMMON} -Build the requested chart from the smallest exact set of returned metric references. -Use multiple references only when the user requested a comparison.`, - - editChart: `${COMMON} -Edit the active chart with exactly the requested operation and the smallest exact set of returned metric references.`, - - clarify: `${COMMON} -Ask one short clarification because the source-derived catalogs did not establish a usable result.`, -}); diff --git a/website_next/ask/tools/refs.js b/website_next/ask/tools/refs.js index 9a4b26485..38c0a2e8e 100644 --- a/website_next/ask/tools/refs.js +++ b/website_next/ask/tools/refs.js @@ -6,6 +6,16 @@ const PREFIX = /** @type {const} */ ({ source: "s", }); +/** @param {string} value */ +function slug(value) { + return value + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, "") + .slice(0, 48); +} + export class AskRefs { /** @type {Map} */ #counts = new Map(); @@ -20,14 +30,16 @@ export class AskRefs { * @param {keyof typeof PREFIX} kind * @param {any} value * @param {string} stableKey + * @param {string} [hint] */ - issue(kind, value, stableKey) { + issue(kind, value, stableKey, hint) { const key = `${kind}:${stableKey}`; const existing = this.#keys.get(key); if (existing) return existing; const count = (this.#counts.get(kind) ?? 0) + 1; - const ref = `${PREFIX[kind]}${count}`; + const suffix = hint ? slug(hint) : ""; + const ref = `${PREFIX[kind]}${count}${suffix ? `_${suffix}` : ""}`; this.#counts.set(kind, count); this.#items.set(ref, { kind, value }); this.#keys.set(key, ref); diff --git a/website_next/ask/tools/render.js b/website_next/ask/tools/render.js index 21fa89eff..a45885da9 100644 --- a/website_next/ask/tools/render.js +++ b/website_next/ask/tools/render.js @@ -1,7 +1,4 @@ import { formatValue } from "./data.js"; -import { focusApiData } from "./api/result.js"; -import { apiRequestWords } from "./api/routing.js"; -import { normalize, tokenAffinity } from "./text.js"; /** * @typedef {Object} MetricRead @@ -11,6 +8,7 @@ import { normalize, tokenAffinity } from "./text.js"; * @property {number | string} start * @property {string | undefined} stamp * @property {unknown[]} values + * @property {string} [requested] */ /** @@ -22,7 +20,6 @@ import { normalize, tokenAffinity } from "./text.js"; */ const SOURCE_URL = "https://github.com/bitcoinresearchkit/brk/blob"; -const MIN_FIELD_AFFINITY = 0.65; /** @param {SourceEvidence} source */ function sourceKey(source) { @@ -60,6 +57,19 @@ export function renderEvidence(evidence) { /** @param {MetricRead[]} results */ export function renderData(results) { return results.map((result) => { + const returnedPosition = result.index === "height" + ? String(result.start) + : result.stamp; + if ( + result.requested && + returnedPosition && + result.requested !== returnedPosition + ) { + const returned = result.index === "height" + ? `block ${returnedPosition}` + : returnedPosition; + return `**${result.label}**: no exact value was returned for ${result.requested}; the server returned ${returned} instead.`; + } if (result.values.length === 1 && typeof result.values[0] === "number") { const position = result.index === "height" ? ` at block ${result.start}` @@ -80,237 +90,3 @@ export function renderData(results) { export function renderApiAnswer(answer, operation) { return `${answer.trim()}\n\nData: \`${operation.method} ${operation.path}\``; } - -/** @param {unknown} value @param {string[]} path */ -function valueAt(value, path) { - let current = value; - for (const key of path) { - if (!current || typeof current !== "object" || !Object.hasOwn(current, key)) { - return undefined; - } - current = /** @type {Record} */ (current)[key]; - } - return current; -} - -/** @param {unknown} value */ -function scalar(value) { - return value === null || - typeof value === "string" || - typeof value === "number" || - typeof value === "boolean"; -} - -/** @param {unknown} value */ -function displayScalar(value) { - if (typeof value === "number") { - return new Intl.NumberFormat("en-US", { maximumFractionDigits: 8 }).format(value); - } - if (typeof value === "string") return value; - return JSON.stringify(value); -} - -/** @param {unknown} data */ -function apiRecords(data) { - if (Array.isArray(data)) return { count: data.length, items: data }; - if ( - data && - typeof data === "object" && - typeof /** @type {{ count?: unknown }} */ (data).count === "number" && - Array.isArray(/** @type {{ sample?: unknown }} */ (data).sample) - ) { - return { - count: /** @type {{ count: number }} */ (data).count, - items: /** @type {{ sample: unknown[] }} */ (data).sample, - }; - } - return { count: undefined, items: [data] }; -} - -/** @param {{ field: { name: string, type: string }, value: unknown }} candidate */ -function renderApiField(candidate) { - const genericTypes = new Set([ - "boolean", - "integer", - "null", - "number", - "object", - "string", - "value", - ]); - const types = candidate.field.type - .split("|") - .map((type) => type.trim()); - const semanticTypes = types.filter((type) => !genericTypes.has(type.toLowerCase())); - const unit = semanticTypes.length === 1 ? ` ${semanticTypes[0]}` : ""; - const label = candidate.field.name.split(".").map(normalize).join(" · "); - return `**${label}**: ${displayScalar(candidate.value)}${unit}`; -} - -/** - * Render a compact schema-derived sample when the user requests a resource - * generally rather than one specific response field. - * - * @param {{ data: unknown, arguments?: Record, operation: { method: string, path: string, summary?: string, response: { fields?: { name: string, type: string }[] } } }} grounding - */ -function renderApiOverview(grounding) { - const data = focusApiData(grounding.data, grounding.arguments); - const { count, items } = apiRecords(data); - const fields = grounding.operation.response.fields ?? []; - const samples = items.slice(0, 3).map((item) => - fields - .map((field) => ({ field, value: valueAt(item, field.name.split(".")) })) - .filter(({ value }) => scalar(value) && value !== null) - .slice(0, 6) - ).filter((sample) => sample.length); - if (!samples.length) return undefined; - - const title = grounding.operation.summary?.trim() || - normalize(grounding.operation.path); - const heading = `**${title}**${count === undefined ? "" : `: ${count} record${count === 1 ? "" : "s"} returned`}.`; - const details = samples.map((sample, index) => { - const label = samples.length > 1 ? `Sample ${index + 1}\n` : ""; - return `${label}${sample.map((candidate) => `- ${renderApiField(candidate)}`).join("\n")}`; - }); - return renderApiAnswer([heading, ...details].join("\n\n"), grounding.operation); -} - -/** - * Render scalar fields selected directly from OpenAPI names and descriptions. - * Equal-scoring fields are returned together rather than asking the model to - * choose, which is both faster and safer for questions such as "which block?". - * Field names, parameter names, and units all come from OpenAPI. - * @param {{ question: string, data: unknown, arguments?: Record, operation: { method: string, path: string, summary?: string, parameters?: { name: string }[], response: { type?: string, fields?: { name: string, type: string, description?: string, ownDescription?: string }[] } } }} grounding - */ -export function renderDirectApiAnswer(grounding) { - const responseFields = grounding.operation.response.fields ?? []; - const normalizedQuestion = normalize(grounding.question); - const totalNoun = - normalizedQuestion.match(/\btotal\s+([a-z0-9]+)\b/)?.[1] ?? - normalizedQuestion.match(/\b([a-z0-9]+)\s+(?:in\s+)?total\b/)?.[1]; - const directTotalFields = totalNoun - ? responseFields.filter((field) => { - const document = new Set( - normalize(`${field.name} ${field.ownDescription ?? field.description ?? ""}`) - .split(" "), - ); - return document.has("total") && - [...document].some((word) => tokenAffinity(word, totalNoun) >= MIN_FIELD_AFFINITY); - }) - : []; - if ( - /\b(?:add(?:ed)?|altogether|combined?|difference|minus|net|plus|subtract(?:ed)?|sum)\b/i - .test(grounding.question) || - totalNoun && directTotalFields.length !== 1 - ) return undefined; - - const data = focusApiData(grounding.data, grounding.arguments); - if (scalar(data) && !responseFields.length) { - const type = grounding.operation.response.type ?? "value"; - const label = normalize(type) || "value"; - return renderApiAnswer(`**${label}**: ${displayScalar(data)}`, grounding.operation); - } - - const words = new Set( - [...apiRequestWords(grounding.question)].filter((word) => word.length >= 3), - ); - if (normalize(grounding.question).includes("how many")) { - words.add("count"); - words.add("number"); - } - const parameters = new Set( - (grounding.operation.parameters ?? []).map(({ name }) => normalize(name)), - ); - const asksForIdentity = words.has("which") || words.has("where"); - const fields = responseFields.map((field, index) => { - const nameTokens = new Set(normalize(field.name).match(/[a-z0-9]+/g) ?? []); - const ownTokens = new Set( - normalize(field.ownDescription ?? field.description ?? "").match(/[a-z0-9]+/g) ?? [], - ); - const tokens = new Set( - normalize(`${field.name} ${field.description ?? ""}`).match(/[a-z0-9]+/g) ?? [], - ); - return { field, index, nameTokens, ownTokens, tokens }; - }); - const frequencies = new Map(); - for (const { tokens } of fields) { - for (const token of tokens) { - frequencies.set(token, (frequencies.get(token) ?? 0) + 1); - } - } - const candidates = fields - .map((field) => { - const path = field.field.name.split("."); - const leaf = path.at(-1) ?? ""; - let matches = 0; - let score = 0; - for (const word of words) { - const match = [...field.tokens] - .map((token) => ({ - token, - affinity: tokenAffinity(word, token), - })) - .sort((left, right) => right.affinity - left.affinity)[0]; - if (!match || match.affinity < MIN_FIELD_AFFINITY) continue; - matches += 1; - const frequency = frequencies.get(match.token) ?? fields.length; - const idf = Math.log((fields.length + 1) / (frequency + 1)) + 1; - const nameMatch = [...field.nameTokens].some((token) => - tokenAffinity(word, token) >= MIN_FIELD_AFFINITY - ); - const ownMatch = [...field.ownTokens].some((token) => - tokenAffinity(word, token) >= MIN_FIELD_AFFINITY - ); - score += idf * (nameMatch ? 3 : ownMatch ? 2 : 1) * match.affinity; - } - if ( - asksForIdentity && - normalize(field.field.type).split(" ").includes("boolean") - ) score *= 0.5; - return { - field: field.field, - index: field.index, - path, - value: valueAt(data, path), - matches, - score, - explicitNameMatches: [...words].filter((word) => - [...field.nameTokens].some((token) => - tokenAffinity(word, token) >= MIN_FIELD_AFFINITY - ) - ).length, - parameter: parameters.has(normalize(leaf)), - unmatchedName: [...field.nameTokens].filter((token) => - ![...words].some((word) => - tokenAffinity(word, token) >= MIN_FIELD_AFFINITY - ) - ).length, - }; - }) - .filter(({ matches, parameter, value }) => - matches > 0 && !parameter && scalar(value) && value !== null - ) - .sort((left, right) => - right.score - left.score || - right.matches - left.matches || - left.unmatchedName - right.unmatchedName - ); - if (!candidates.length) return renderApiOverview(grounding); - - const explicit = candidates.filter(({ explicitNameMatches }) => explicitNameMatches > 0); - const selected = ( - /\band\b/i.test(grounding.question) && explicit.length > 1 - ? explicit - : candidates.filter(({ score, matches, unmatchedName }) => - score === candidates[0].score && - matches === candidates[0].matches && - (matches === 1 || unmatchedName === candidates[0].unmatchedName) - ) - ) - .slice(0, 6) - .sort((left, right) => left.index - right.index); - const answer = selected.length === 1 - ? renderApiField(selected[0]) - : selected.map((candidate) => `- ${renderApiField(candidate)}`).join("\n"); - return renderApiAnswer(answer, grounding.operation); -} diff --git a/website_next/ask/tools/schemas.js b/website_next/ask/tools/schemas.js deleted file mode 100644 index 9b2fcd82a..000000000 --- a/website_next/ask/tools/schemas.js +++ /dev/null @@ -1,170 +0,0 @@ -/** @param {Record} properties @param {string[]} required */ -function actionTool(properties, required) { - return { - type: "function", - function: { - name: "next_action", - description: "Choose exactly one allowed next step.", - parameters: { - type: "object", - properties, - required, - additionalProperties: false, - }, - }, - }; -} - -/** @param {boolean} hasActiveChart @param {boolean} hasPrevious */ -export function searchTool(hasActiveChart = false, hasPrevious = false) { - return actionTool({ - action: { type: "string", enum: ["search"] }, - context: { - type: "string", - enum: hasPrevious - ? ["new_topic", "reuse_previous", "extend_previous"] - : ["new_topic"], - description: "Reuse the previous verified topic for dependent follow-ups. Extend it only when the user adds another distinct metric. Otherwise start a new topic.", - }, - queries: { - type: "array", - minItems: 1, - maxItems: 4, - items: { type: "string" }, - description: "One terse catalog-style Bitcoin metric or technical noun phrase per distinct topic. Translate the user's meaning. Never copy a question or include request verbs, pronouns, time words, or punctuation. For X vs Y, return separate complete X and Y metric phrases.", - }, - cardinality: { - type: "string", - enum: ["single", "multiple"], - description: "Use multiple whenever the user requests a comparison or more than one distinct metric, even if queries accidentally contains one item.", - }, - outcome: { - type: "string", - enum: [ - "read_requested_value", - "read_api", - "build_requested_chart", - ...(hasActiveChart ? ["edit_existing_chart"] : []), - "explain_from_verified_facts", - "answer_general", - "clarify_request", - ], - }, - clarification: { - type: "string", - description: "Only for clarify_request: one short question that distinguishes the materially different interpretations.", - }, - }, ["action", "outcome"]); -} - -/** @param {string[]} queries */ -export function rewriteTool(queries) { - return actionTool({ - action: { type: "string", enum: ["rewrite"] }, - queries: { - type: "array", - minItems: queries.length, - maxItems: queries.length, - items: { type: "string" }, - description: `Rewrite each input independently, in the same order, without merging them: ${queries.map((query, index) => `${index + 1}=${query}`).join("; ")}.`, - }, - }, ["action", "queries"]); -} - -/** @param {{ ref: string, label: string, operation: import("./api/index.js").ApiOperation }[]} options */ -export function apiResolveTool(options) { - const parameters = new Map(); - for (const { operation } of options) { - for (const parameter of operation.parameters) { - const current = parameters.get(parameter.name); - const descriptions = [ - current?.description, - `${parameter.in}${parameter.required ? ", required" : ""} for ${operation.path}${parameter.description ? `: ${parameter.description}` : ""}`, - ].filter(Boolean); - parameters.set(parameter.name, { - type: "string", - description: [...new Set(descriptions)].join(" "), - }); - } - } - return actionTool({ - action: { type: "string", enum: ["call_api", "clarify"] }, - ref: { - type: "string", - enum: options.map(({ ref }) => ref), - description: `Read-only operations: ${options.map(({ ref, label, operation }) => { - const params = operation.parameters - .map((parameter) => `${parameter.name}${parameter.required ? "*" : ""}`) - .join(", "); - return `${ref}=${label} [${params || "no parameters"}]`; - }).join("; ")}`, - }, - arguments: { - type: "object", - properties: Object.fromEntries(parameters), - additionalProperties: false, - description: "Arguments copied from the user's request. Include every required parameter for the selected operation.", - }, - text: { type: "string", description: "For clarify only: one short question." }, - }, ["action"]); -} - -/** - * @param {{ ref: string, label: string }[]} options - * @param {string} outcome - * @param {number} [maxItems] - */ -export function resolveTool(options, outcome, maxItems = 3) { - const refs = { - type: "array", - minItems: 1, - maxItems, - items: { type: "string", enum: options.map(({ ref }) => ref) }, - description: `Available references: ${options.map(({ ref, label }) => `${ref}=${label}`).join("; ")}`, - }; - - if (outcome === "explain_from_verified_facts") { - return actionTool({ - action: { type: "string", enum: ["answer", "clarify"] }, - refs, - text: { type: "string", description: "For clarify only: one short question." }, - }, ["action"]); - } - - if (outcome === "read_requested_value") { - return actionTool({ - action: { type: "string", enum: ["read_data", "clarify"] }, - refs, - mode: { type: "string", enum: ["latest", "at", "range"] }, - index: { type: "string", description: "Index such as height or day1." }, - at: { type: "string", description: "Block height or date for at mode." }, - start: { type: "string" }, - end: { type: "string" }, - points: { type: "integer", minimum: 1, maximum: 120 }, - text: { type: "string", description: "For clarify only: one short question." }, - }, ["action"]); - } - - const editing = outcome === "edit_existing_chart"; - return actionTool({ - action: { - type: "string", - enum: [editing ? "edit_chart" : "build_chart", "clarify"], - }, - refs, - title: { type: "string" }, - operation: { - type: "string", - enum: ["add", "remove", "replace"], - description: "For edit_chart, make exactly the requested change.", - }, - text: { type: "string", description: "For clarify only: one short question." }, - }, ["action"]); -} - -export function clarifyTool() { - return actionTool({ - action: { type: "string", enum: ["clarify"] }, - text: { type: "string", description: "One necessary clarification question." }, - }, ["action", "text"]); -} diff --git a/website_next/ask/tools/session.js b/website_next/ask/tools/session.js deleted file mode 100644 index 223681128..000000000 --- a/website_next/ask/tools/session.js +++ /dev/null @@ -1,1542 +0,0 @@ -import { createChartArtifact } from "./chart.js"; -import { resolveChartUnit } from "./chart/units.js"; -import { apiByKey, searchApi } from "./api/index.js"; -import { executeApi } from "./api/execute.js"; -import { - apiResponseMatchCount, - apiOperationMatchCount, - argumentAffinity, - directApiCandidate, - literalArguments, - literalType, - matchesApiIntent, - matchesApiResponse, - reusableArguments, -} from "./api/routing.js"; -import { readMetric } from "./data.js"; -import { directChartCommand } from "./direct/chart.js"; -import { directEvidenceFocus } from "./direct/evidence.js"; -import { - completeComparisonQueries, - directReadAction, - evidenceFocus, - isDirectDefinition, - isDirectValueFollowup, - isDirectValueRequest, - isExplicitComparison, - mayRequestMultiple, - referencesPlural, - referencesPrevious, - referencesSingular, -} from "./direct/request.js"; -import { - latestApiContext, - latestChart, - latestKnowledgeContext, - latestMetricPaths, - latestSourceContext, - recentMetricPaths, -} from "./history.js"; -import { searchLearn } from "./learn.js"; -import { - metricByName, - metricVariants, - metricsByPaths, - mentionedMetrics, - searchMetrics, -} from "./metrics/index.js"; -import { - balancedOptions, - coordinatedMetrics, - exactTopicMetrics, - MAX_OPTIONS, - mergeMetricGroups, - uniqueMetricOptions, -} from "./metrics/options.js"; -import { canonicalMetricQuery } from "./metrics/language.js"; -import { ASK_STAGE_PROMPTS } from "./prompts.js"; -import { AskRefs } from "./refs.js"; -import { renderData, renderEvidence } from "./render.js"; -import { - directSourceFact, - hasDirectSourceComputation, - sourceFocus, -} from "./source/answer.js"; -import { - apiResolveTool, - clarifyTool, - resolveTool, - rewriteTool, - searchTool, -} from "./schemas.js"; -import { normalize } from "./text.js"; - -const MAX_API_OPTIONS = 6; -const MAX_API_HINTS = 24; - -/** @typedef {import("../storage.js").ChartArtifact} ChartArtifact */ - -/** - * @typedef {Object} MetricOption - * @property {string} ref - * @property {string} label - * @property {any} metric - */ - -/** @param {unknown} value @param {string} name */ -function requiredString(value, name) { - if (typeof value !== "string" || !value.trim()) throw new Error(`${name} is required`); - return value.trim(); -} - -/** @param {unknown} value @param {string} name */ -function requiredStrings(value, name) { - if ( - !Array.isArray(value) || - !value.length || - value.some((item) => typeof item !== "string" || !item.trim()) - ) throw new Error(`${name} are required`); - return [...new Set(value.map((item) => item.trim()))]; -} - -/** @param {unknown} value */ -function uniqueRefs(value) { - if (!Array.isArray(value) || !value.length || value.some((ref) => typeof ref !== "string")) { - throw new Error("One or more returned references are required"); - } - return [...new Set(value)]; -} - -/** @param {string} value */ -function label(value) { - return value.replaceAll("_", " "); -} - -/** @param {string} formulaMetric @param {string} metricName */ -function formulaMatchesMetric(formulaMetric, metricName) { - const formula = normalize(formulaMetric); - const metric = normalize(metricName); - return metric === formula || metric.endsWith(` ${formula}`); -} - -/** @param {any} formula @param {any} metric */ -function metricFormulaAnswer(formula, metric) { - if (!formula.fact.summary) return formula.answer; - const name = label(metric.name); - const displayName = !name.includes(" ") && name.length <= 4 - ? name.toUpperCase() - : `${name[0].toUpperCase()}${name.slice(1)}`; - const unit = metric.suggestedUnit && metric.suggestedUnit !== "number" - ? ` (${String(metric.suggestedUnit).toUpperCase()})` - : ""; - return `${displayName}${unit} ${formula.fact.summary}.`; -} - -/** @param {any} formula */ -function formulaCitation(formula) { - return { - revision: formula.revision, - path: formula.fact.path, - startLine: formula.fact.line, - }; -} - -export class AskToolSession { - /** @param {import("./source/index.js").AskSource} source */ - constructor(source) { - this.source = source; - } - - source; - refs = new AskRefs(); - request = ""; - /** @type {string[]} */ - previousTopics = []; - /** @type {any[]} */ - previousMetrics = []; - /** @type {any[]} */ - recentMetrics = []; - /** @type {any[]} */ - contextMetrics = []; - query = ""; - /** @type {string[]} */ - queries = []; - outcome = ""; - focus = "definition"; - stage = "search"; - directMatch = false; - rewritten = false; - rewriteChanged = false; - comparison = false; - /** @type {any} */ - observation; - /** @type {any[]} */ - options = []; - /** @type {MetricOption[]} */ - metricOptions = []; - /** @type {{ ref: string, label: string, operation: import("./api/index.js").ApiOperation }[]} */ - apiOptions = []; - /** @type {import("./api/index.js").ApiOperation[]} */ - apiHints = []; - /** @type {import("./api/index.js").ApiOperation[]} */ - apiCandidates = []; - /** @type {import("./api/index.js").ApiOperation | undefined} */ - directApiHint; - verifyDirectApiIntent = false; - /** @type {{ operation: import("./api/index.js").ApiOperation, arguments: Record } | undefined} */ - previousApi; - reuseApiContext = false; - /** @type {any} */ - formula; - sourceSearched = false; - /** @type {import("../storage.js").SourceContext[]} */ - previousSource = []; - /** @type {import("../storage.js").KnowledgeContext | undefined} */ - previousKnowledge; - requiresTools = false; - /** @type {ChartArtifact | undefined} */ - activeChart; - - /** @param {string} request @param {any[]} history @param {() => void} onProgress */ - async begin(request, history, onProgress) { - this.refs = new AskRefs(); - this.request = request.trim(); - this.query = ""; - this.queries = []; - this.outcome = ""; - this.focus = "definition"; - this.stage = "search"; - this.directMatch = false; - this.rewritten = false; - this.rewriteChanged = false; - this.comparison = false; - this.contextMetrics = []; - this.observation = undefined; - this.options = []; - this.metricOptions = []; - this.apiOptions = []; - this.apiHints = []; - this.apiCandidates = []; - this.directApiHint = undefined; - this.verifyDirectApiIntent = false; - this.reuseApiContext = false; - this.formula = undefined; - this.sourceSearched = false; - this.previousSource = latestSourceContext(history) ?? []; - this.previousKnowledge = latestKnowledgeContext(history); - this.requiresTools = false; - this.activeChart ??= latestChart(history); - - const apiContext = latestApiContext(history); - if (apiContext?.key) { - const operation = await apiByKey(apiContext.key); - this.previousApi = operation - ? { operation, arguments: apiContext.arguments ?? {} } - : undefined; - } else { - this.previousApi = undefined; - } - - const focusPaths = latestMetricPaths(history); - const literals = literalArguments(this.request); - const hasResourceIdentifier = literals.some((value) => literalType(value) === "string"); - const dependsOnMetric = Boolean(focusPaths?.length) && - !this.previousApi && - referencesPrevious(this.request) && - !hasResourceIdentifier; - let prefersMetricTool = dependsOnMetric || - Boolean(directChartCommand(this.request, Boolean(this.activeChart))); - const dependsOnPrevious = Boolean(this.previousApi) && referencesPrevious(this.request); - const previousApiContext = this.previousApi - ? [ - this.previousApi.operation.summary || this.previousApi.operation.label, - ...this.previousApi.operation.parameters.flatMap((parameter) => [ - parameter.name, - parameter.type, - parameter.description, - ]), - ].filter(Boolean).join(" ") - : ""; - const apiQuery = dependsOnPrevious - ? `${this.request} ${previousApiContext}` - : this.request; - this.apiHints = await searchApi([apiQuery], MAX_API_HINTS, onProgress).catch(() => []); - this.apiCandidates = !prefersMetricTool && literals.length - ? this.apiHints.filter((operation) => - argumentAffinity(literals, operation, this.request) >= 0 && - (operation.matchedTerms ?? 0) > 0 - ) - : []; - if ( - !prefersMetricTool && - !this.previousApi && - isDirectValueRequest(this.request) && - literals.some((value) => literalType(value) !== "string") - ) { - const pointMetrics = await mentionedMetrics(this.request, onProgress); - if (pointMetrics.length) { - prefersMetricTool = true; - this.apiCandidates = []; - } - } - this.directApiHint = prefersMetricTool - ? undefined - : directApiCandidate(this.apiHints, literals, this.request); - this.verifyDirectApiIntent = Boolean( - this.directApiHint && - !this.previousApi && - !literals.length && - this.apiHints[0]?.key !== this.directApiHint.key, - ); - this.requiresTools = Boolean(this.directApiHint) || this.apiCandidates.length > 0; - - if ( - !prefersMetricTool && - this.previousApi && - ( - referencesPrevious(this.request) || - matchesApiResponse(this.request, this.previousApi.operation) - ) - ) { - this.requiresTools = true; - const contextual = this.apiHints - .filter((operation) => - (operation.matchedTerms ?? 0) >= 2 && - matchesApiIntent(this.request, operation) && - reusableArguments(operation, this.previousApi) !== undefined - ) - .sort((left, right) => - apiResponseMatchCount(this.request, right) - - apiResponseMatchCount(this.request, left) || - (right.matchedTerms ?? 0) - (left.matchedTerms ?? 0) || - (right.score ?? 0) - (left.score ?? 0) - )[0]; - if (contextual) { - this.directApiHint = contextual; - } else if (matchesApiResponse(this.request, this.previousApi.operation)) { - this.directApiHint = this.previousApi.operation; - } - } - - if (focusPaths === undefined) return; - const recentPaths = recentMetricPaths(history); - const currentFocus = this.previousMetrics.map((metric) => metric.path); - const currentRecent = this.recentMetrics.map((metric) => metric.path); - if ( - focusPaths.length === currentFocus.length && - focusPaths.every((path, index) => path === currentFocus[index]) && - recentPaths.length === currentRecent.length && - recentPaths.every((path, index) => path === currentRecent[index]) - ) { - return; - } - const paths = [...new Set([...focusPaths, ...recentPaths])]; - const metrics = paths.length ? await metricsByPaths(paths, onProgress) : []; - const byPath = new Map(metrics.map((metric) => [metric.path, metric])); - this.previousMetrics = focusPaths.map((path) => byPath.get(path)).filter(Boolean); - this.previousTopics = this.previousMetrics.map((metric) => label(metric.name)); - this.recentMetrics = recentPaths.map((path) => byPath.get(path)).filter(Boolean); - } - - /** @param {(status: string) => void} onStatus @param {AbortSignal} signal */ - async tryDirect(onStatus, signal) { - if (this.directApiHint) { - this.outcome = "read_api"; - this.queries = [this.request]; - this.apiOptions = [{ - ref: this.refs.issue("api", this.directApiHint, this.directApiHint.key), - label: `${this.directApiHint.method} ${this.directApiHint.path} — ${this.directApiHint.label}`, - operation: this.directApiHint, - }]; - try { - const direct = await this.tryDirectApi(onStatus, signal); - if (direct) return direct; - } catch { - this.requiresTools = true; - this.apiOptions = []; - } - } - - if (this.apiCandidates.length) { - this.outcome = "read_api"; - this.queries = [this.request]; - await this.searchApiOperations(onStatus); - return undefined; - } - - const evidence = directEvidenceFocus(this.request); - const chart = evidence - ? undefined - : directChartCommand(this.request, Boolean(this.activeChart)); - if (chart) { - if (chart.kind === "missing") { - return { - output: "Which metric should I chart?", - artifacts: [], - }; - } - if (chart.kind === "missing_add") { - const added = await mentionedMetrics( - this.request, - () => onStatus("Indexing metrics…"), - ); - const metrics = [...new Map( - [...this.previousMetrics, ...added].map((metric) => [metric.path, metric]), - ).values()]; - if (metrics.length > 1) { - this.rememberMetricValues(metrics); - return { - output: `Do you want me to chart ${metrics.map((metric) => label(metric.name)).join(" and ")} together?`, - artifacts: [], - }; - } - return { - output: "What should I add it to?", - artifacts: [], - }; - } - if (chart.kind === "style") { - this.outcome = "edit_existing_chart"; - onStatus("Updating chart…"); - const result = this.restyleChart(chart); - return { output: result.output, artifacts: result.artifacts }; - } - - let metrics = await mentionedMetrics( - this.request, - () => onStatus("Indexing metrics…"), - ); - if (metrics.length < 2 && mayRequestMultiple(this.request)) { - metrics = await coordinatedMetrics( - this.request, - () => onStatus("Searching metrics…"), - ); - } - if (!metrics.length) { - if (this.previousTopics.length > 1 && referencesSingular(this.request)) { - return undefined; - } - metrics = referencesPlural(this.request) && this.recentMetrics.length - ? this.recentMetrics - : this.previousMetrics.length - ? this.previousMetrics - : await exactTopicMetrics( - this.previousTopics, - () => onStatus("Indexing metrics…"), - ); - } - - if (metrics.length && (!mayRequestMultiple(this.request) || metrics.length > 1)) { - const refs = metrics.map((metric) => this.refs.issue("metric", metric, metric.path)); - this.outcome = chart.kind === "edit" - ? "edit_existing_chart" - : "build_requested_chart"; - onStatus("Building chart…"); - try { - const result = this.buildChart({ refs, operation: chart.operation }); - return { output: result.output, artifacts: result.artifacts }; - } catch (error) { - if (chart.kind !== "edit") throw error; - return { - output: error instanceof Error ? error.message : String(error), - artifacts: [], - }; - } - } - this.requiresTools = true; - } - - if (evidence) { - this.requiresTools = true; - let metrics = await mentionedMetrics( - this.request, - () => onStatus("Indexing metrics…"), - ); - if (!metrics.length && this.previousMetrics.length === 1 && referencesPrevious(this.request)) { - metrics = this.previousMetrics; - } - - if (metrics.length) { - if (evidence === "implementation" && metrics.length !== 1) { - metrics = []; - } - if (evidence === "variants") { - const supported = await Promise.all( - metrics.map(async (metric) => - await metricVariants(metric, this.request) ? metric : undefined - ), - ); - metrics = supported.filter((metric) => metric !== undefined); - if (!metrics.length) return undefined; - } - if (evidence === "implementation" && metrics.length) { - onStatus("Searching source…"); - this.formula = await this.source.explain( - [this.request, ...metrics.map((metric) => metric.name)].join("\n"), - ({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`), - ); - if ( - !this.formula || - !metrics.some((metric) => - formulaMatchesMetric(this.formula.fact.metric, metric.name) - ) - ) { - metrics = []; - } - } - - if (metrics.length) { - this.focus = evidence; - const refs = metrics.map((metric) => this.refs.issue("metric", metric, metric.path)); - onStatus("Inspecting results…"); - const result = await this.inspect(refs); - return { output: result.output, artifacts: [] }; - } - } - if (evidence === "implementation") { - if ( - this.previousSource.length && - directSourceFact(this.request, this.previousSource) - ) { - return { - output: "", - artifacts: [], - grounding: { - question: this.request, - excerpts: this.previousSource, - }, - }; - } - onStatus("Searching source…"); - const result = await this.source.search( - this.request, - undefined, - ({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`), - ); - if (result.matches.length) { - const [match] = result.matches; - const excerpt = { ...match, revision: result.revision }; - if (hasDirectSourceComputation(this.request, [excerpt])) { - const focus = sourceFocus(this.request, [excerpt]); - return { - output: "", - artifacts: [], - grounding: { - question: this.request, - excerpts: [{ ...excerpt, ...(focus ? { focus } : {}) }], - }, - }; - } - const options = result.matches.slice(0, 6).map((/** @type {any} */ match) => { - const value = { ...match, revision: result.revision }; - return { - ref: this.refs.issue("source", value, `${match.path}:${match.startLine}`), - kind: "source", - label: `${match.path}:${match.startLine}`, - detail: match.content.slice(0, 180), - }; - }); - this.outcome = "explain_from_verified_facts"; - this.stage = "resolve"; - this.options = options; - this.observation = { options }; - this.requiresTools = true; - return undefined; - } - } - this.requiresTools = true; - } - - if ( - this.previousSource.length && - ( - directSourceFact(this.request, this.previousSource) || - referencesPrevious(this.request) && - /^(?:are|does|how|is|what|which|why)\b/i.test(this.request.trim()) - ) && - !isDirectValueRequest(this.request) - ) { - this.requiresTools = true; - return { - output: "", - artifacts: [], - grounding: { - question: this.request, - excerpts: this.previousSource, - }, - }; - } - - if (isExplicitComparison(this.request)) { - let metrics = await mentionedMetrics( - this.request, - () => onStatus("Indexing metrics…"), - ); - if (referencesPrevious(this.request)) { - metrics = [...this.previousMetrics, ...metrics]; - } - metrics = [...new Map(metrics.map((metric) => [metric.path, metric])).values()]; - if (metrics.length >= 2) { - onStatus("Searching source…"); - const formulas = ( - await Promise.all(metrics.map((metric) => - this.source.explain( - `${this.request}\n${metric.name}`, - ({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`), - ) - )) - ).filter(Boolean); - if (formulas.length) { - const facts = formulas.map((formula) => { - const metric = metrics.find((item) => - formulaMatchesMetric(formula.fact.metric, item.name) - ); - return metric ? metricFormulaAnswer(formula, metric) : formula.answer; - }); - for (const formula of formulas) { - if (formula.fact.kind !== "ratio") continue; - const denominator = normalize(formula.fact.denominator ?? ""); - const related = metrics.find((metric) => - normalize(metric.name) === denominator || - normalize(metric.name).endsWith(` ${denominator}`) - ); - if (related?.suggestedUnit) { - facts.push( - `${label(related.name)} is the ${String(related.suggestedUnit).toUpperCase()} denominator in ${formula.fact.metric.toUpperCase()}.`, - ); - } - } - this.rememberMetricValues(metrics); - return { - output: renderEvidence({ - facts, - sources: formulas.map(formulaCitation), - excerpts: [], - }), - artifacts: [], - }; - } - this.rememberMetricValues(metrics); - return { - output: "Do you want their latest values, definitions, or a chart?", - artifacts: [], - }; - } - } - - if ( - isDirectValueRequest(this.request) || - this.previousTopics.length === 1 && isDirectValueFollowup(this.request) - ) { - const action = directReadAction(this.request); - if (action) { - let metrics = await mentionedMetrics( - this.request, - () => onStatus("Indexing metrics…"), - ); - if (!metrics.length) { - metrics = this.previousMetrics.length - ? this.previousMetrics - : await exactTopicMetrics( - this.previousTopics, - () => onStatus("Indexing metrics…"), - ); - } - - if (metrics.length > 1 && referencesSingular(this.request)) { - this.requiresTools = true; - return undefined; - } - if (metrics.length) { - onStatus("Reading data…"); - const results = await Promise.all(metrics.map((metric) => readMetric(metric, action))); - this.rememberMetricValues(metrics); - return { output: renderData(results), artifacts: [] }; - } - this.requiresTools = true; - } - } - - if (mayRequestMultiple(this.request)) { - const metrics = await mentionedMetrics( - this.request, - () => onStatus("Indexing metrics…"), - ); - if (metrics.length >= 2) { - this.rememberMetricValues(metrics); - return { - output: "Do you want their latest values, definitions, or a chart?", - artifacts: [], - }; - } - } - - if ( - this.previousKnowledge && - ( - referencesPrevious(this.request) || - /\b(?:analogy|trade-?off)\b/i.test(this.request) - ) - ) { - if (/\btrade-?off\b/i.test(this.request)) { - return { - output: `Which tradeoff about ${this.previousKnowledge.title} do you mean?`, - artifacts: [], - knowledgeContext: this.previousKnowledge, - }; - } - return { - output: "", - artifacts: [], - knowledgeGrounding: { - question: this.request, - context: this.previousKnowledge, - }, - }; - } - - if (!isDirectDefinition(this.request)) return undefined; - - const explicitlyNamesMetric = /\b(?:indicator|metric|series)\b/i.test(this.request); - const mentioned = await mentionedMetrics( - this.request, - () => onStatus("Indexing metrics…"), - ); - if (mentioned.length > 1) return undefined; - let [metric] = mentioned; - if (!metric && this.previousMetrics.length === 1 && referencesPrevious(this.request)) { - [metric] = this.previousMetrics; - } - if (!metric && !explicitlyNamesMetric) { - const [guide] = await searchLearn(this.request, 1); - if (guide?.titleCoverage === 1 && guide.description) { - const guideMetrics = ( - await Promise.all( - guide.series.map((/** @type {any} */ series) => metricByName(series.name)), - ) - ).filter(Boolean); - if (guideMetrics.length) this.rememberMetricValues(guideMetrics); - return { - output: guide.description, - artifacts: [], - knowledgeContext: { - title: guide.title, - description: guide.description, - }, - }; - } - return undefined; - } - if (!metric) { - [metric] = await searchMetrics( - [this.request], - 1, - [], - () => onStatus("Indexing metrics…"), - ); - } - if (!metric) { - this.requiresTools = true; - return undefined; - } - - onStatus("Searching source…"); - const formula = await this.source.explain( - `${this.request}\n${metric.name}`, - ({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`), - ); - if (!formula || !formulaMatchesMetric(formula.fact.metric, metric.name)) { - const result = await this.source.search( - `${metric.name} compute calculation definition state`, - undefined, - ({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`), - ); - const [match] = result.matches; - if (!match) { - this.requiresTools = true; - return undefined; - } - - onStatus("Inspecting results…"); - const excerpt = await this.source.read( - match.path, - match.startLine, - match.endLine, - ); - this.rememberMetricValues([metric]); - return { - output: "", - artifacts: [], - grounding: { - question: this.request, - metric: { - name: label(metric.name), - path: metric.path, - unit: metric.suggestedUnit, - }, - excerpts: [excerpt], - }, - }; - } - - this.rememberMetricValues([metric]); - return { - output: renderEvidence({ - facts: [metricFormulaAnswer(formula, metric)], - sources: [formulaCitation(formula)], - excerpts: [], - }), - artifacts: [], - }; - } - - async tool() { - if (this.stage === "search") { - return searchTool( - Boolean(this.activeChart), - this.previousTopics.length > 0 || Boolean(this.previousApi), - ); - } - if (this.stage === "rewrite") return rewriteTool(this.queries); - if (this.stage === "resolve") { - if (this.outcome === "read_api") return apiResolveTool(this.apiOptions); - const maxItems = this.formula - ? 1 - : this.outcome === "explain_from_verified_facts" - ? Math.min(this.queries.length, 3) - : this.comparison - ? this.outcome === "read_requested_value" ? 3 : 6 - : 1; - const options = this.outcome === "explain_from_verified_facts" - ? this.options - : this.metricOptions; - return resolveTool(options, this.outcome, Math.max(1, maxItems)); - } - return clarifyTool(); - } - - instruction() { - if (this.stage === "search") { - const previous = this.previousTopics.length - ? `\nPrevious verified topic${this.previousTopics.length === 1 ? "" : "s"}: ${this.previousTopics.join(", ")}. Reuse only when the newest request depends on it.` - : ""; - const previousApi = this.previousApi - ? `\nPrevious verified API resource: ${this.previousApi.operation.label} (${this.previousApi.operation.key}), arguments ${JSON.stringify(this.previousApi.arguments)}. Reuse only for a dependent follow-up on that resource.` - : ""; - const chart = this.activeChart - ? `\nActive chart: ${this.activeChart.chart.title}; series: ${this.activeChart.chart.series.map((item) => item.label).join(", ")}.` - : ""; - return `${ASK_STAGE_PROMPTS.search}${previous}${previousApi}${chart}`; - } - if (this.stage === "rewrite") return ASK_STAGE_PROMPTS.rewrite; - if (this.stage === "resolve") { - if (this.outcome === "read_api") return ASK_STAGE_PROMPTS.api; - if (this.outcome === "explain_from_verified_facts") { - return this.directMatch - ? `${ASK_STAGE_PROMPTS.explain}\nA trusted direct match exists. Use only recommendedRefs.` - : ASK_STAGE_PROMPTS.explain; - } - if (this.outcome === "read_requested_value") return ASK_STAGE_PROMPTS.read; - return this.outcome === "edit_existing_chart" - ? ASK_STAGE_PROMPTS.editChart - : ASK_STAGE_PROMPTS.chart; - } - return ASK_STAGE_PROMPTS.clarify; - } - - /** @param {(status: string) => void} onStatus */ - async search(onStatus) { - if (this.outcome === "read_api") return await this.searchApiOperations(onStatus); - - const [globalMetrics, rawMetrics, guideGroups] = await Promise.all([ - searchMetrics( - this.queries, - MAX_OPTIONS, - [], - () => onStatus("Indexing metrics…"), - ), - searchMetrics( - [this.request], - MAX_OPTIONS, - [], - () => onStatus("Indexing metrics…"), - ), - Promise.all(this.queries.map((query) => searchLearn(query, MAX_OPTIONS))), - ]); - const foundMetrics = mergeMetricGroups([globalMetrics, rawMetrics]); - const contextualMetrics = this.contextMetrics.map((metric) => ({ - ...metric, - matchedQuery: label(metric.name), - score: 2_000, - })); - const metrics = [...new Map( - [...foundMetrics, ...contextualMetrics].map((metric) => [metric.path, metric]), - ).values()]; - const metricNames = new Set(metrics.map((metric) => metric.name)); - const guides = [...new Map( - guideGroups.flat().map((/** @type {any} */ guide) => [guide.breadcrumbs.join("/"), guide]), - ).values()].filter((/** @type {any} */ guide) => - this.outcome === "explain_from_verified_facts" || - guide.series.some((/** @type {any} */ series) => metricNames.has(series.name)) - ); - /** @type {any[]} */ - let sources = []; - - const hasExactMetric = foundMetrics.some((metric) => - normalize(metric.name) === normalize(metric.matchedQuery ?? "") - ); - const hasGroundedSubject = hasExactMetric || - this.contextMetrics.length > 0 || - this.rewritten && this.rewriteChanged; - - if ( - this.outcome === "explain_from_verified_facts" && - this.focus !== "variants" && - !this.sourceSearched && - (this.focus === "implementation" || hasGroundedSubject) - ) { - this.sourceSearched = true; - onStatus("Searching source…"); - const originalMetric = foundMetrics.find((metric) => - normalize(metric.matchedQuery ?? "") === normalize(this.request) - ); - this.formula = await this.source.explain( - [this.request, originalMetric?.name, ...this.queries].filter(Boolean).join("\n"), - ({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`), - ); - if (!this.formula) { - const result = await this.source.search( - this.queries.join(" "), - undefined, - ({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`), - ); - sources = result.matches.map((/** @type {any} */ source) => ({ - ...source, - revision: result.revision, - })); - } else { - const formulaMetric = await metricByName(this.formula.fact.metric); - if (formulaMetric && !metrics.some((metric) => metric.path === formulaMetric.path)) { - metrics.unshift({ - ...formulaMetric, - matchedQuery: label(this.formula.fact.metric), - score: 2_000, - }); - } - } - } - - /** @type {any[]} */ - const ranked = []; - for (const metric of metrics) { - const ref = this.refs.issue("metric", metric, metric.path); - ranked.push({ - ref, - kind: "metric", - label: label(metric.name), - detail: metric.path, - matchedQuery: metric.matchedQuery, - score: metric.score, - }); - } - for (const guide of guides) { - const key = guide.breadcrumbs.join("/"); - const ref = this.refs.issue("guide", guide, key); - ranked.push({ - ref, - kind: "guide", - label: guide.title, - detail: guide.description, - score: guide.score, - }); - } - for (const source of sources) { - const ref = this.refs.issue("source", source, `${source.path}:${source.startLine}`); - ranked.push({ - ref, - kind: "source", - label: `${source.path}:${source.startLine}`, - detail: source.content, - score: source.score, - }); - } - if (this.formula && !metrics.length) { - const ref = this.refs.issue("fact", this.formula, this.formula.fact.metric); - ranked.push({ - ref, - kind: "fact", - label: label(this.formula.fact.metric), - detail: this.formula.answer, - score: 1_000, - }); - } - - this.options = balancedOptions(ranked); - if (!this.options.length) { - this.stage = "clarify"; - this.observation = { noMatch: true }; - return; - } - - const normalizedQueries = this.queries.map((query) => normalize(canonicalMetricQuery(query))); - const exactCandidates = this.options.filter((option) => - option.kind === "metric" && - normalize(option.label) === normalize(option.matchedQuery ?? "") - ); - const exact = exactCandidates.filter((option) => { - const optionLabel = normalize(option.label); - return !exactCandidates.some((candidate) => { - const candidateLabel = normalize(candidate.label); - return candidateLabel !== optionLabel && candidateLabel.includes(optionLabel); - }); - }); - const exactByQuery = normalizedQueries.map((query) => - exact.filter((option) => normalize(option.matchedQuery ?? "") === query) - ); - this.directMatch = exactByQuery.every((matches) => matches.length === 1); - const directOptions = exactByQuery.flat(); - const trusted = this.directMatch || - Boolean(this.formula) || - sources.length > 0; - if (!trusted) { - if (!this.rewritten) { - this.stage = "rewrite"; - this.observation = { unmatchedQueries: this.queries }; - return; - } - return { - done: true, - output: "I couldn't find a matching series. What metric should I use instead?", - }; - } - - if ( - this.directMatch && - this.outcome === "explain_from_verified_facts" && - this.focus === "variants" - ) { - onStatus("Inspecting results…"); - return this.inspect(directOptions.map(({ ref }) => ref)); - } - if (this.directMatch && this.outcome === "read_requested_value") { - const action = directReadAction(this.request); - if (action) { - const refs = directOptions.map(({ ref }) => ref); - this.rememberMetrics(refs); - onStatus("Reading data…"); - return this.read({ refs, ...action }); - } - } - if (this.directMatch && this.outcome === "build_requested_chart") { - const refs = directOptions.map(({ ref }) => ref); - onStatus("Building chart…"); - return this.buildChart({ refs }); - } - - this.stage = "resolve"; - const formulaOption = this.formula - ? this.options.find((option) => - option.kind === "metric" && - normalize(option.label) === normalize(this.formula.fact.metric) - ) - : undefined; - if (this.outcome === "explain_from_verified_facts") { - if (formulaOption) return this.inspect([formulaOption.ref]); - - this.observation = { - recommendedRefs: (directOptions.length ? directOptions : this.options) - .slice(0, 3) - .map(({ ref }) => ref), - options: this.options, - }; - return; - } - - await this.prepareMetricOptions(); - } - - /** @param {(status: string) => void} onStatus */ - async searchApiOperations(onStatus) { - onStatus("Searching API…"); - const values = literalArguments(this.request); - const found = await searchApi( - [this.request, ...this.queries], - MAX_OPTIONS, - () => onStatus("Indexing API…"), - ); - const candidates = [...new Map( - [...this.apiHints, ...found].map((operation) => [operation.key, operation]), - ).values()]; - candidates.sort((left, right) => { - const leftRequired = left.parameters.filter((parameter) => parameter.required).length; - const rightRequired = right.parameters.filter((parameter) => parameter.required).length; - return apiOperationMatchCount(this.request, right) - - apiOperationMatchCount(this.request, left) || - argumentAffinity(values, right, this.request) - - argumentAffinity(values, left, this.request) || - Number(rightRequired === values.length) - Number(leftRequired === values.length) || - (right.score ?? 0) - (left.score ?? 0); - }); - const operations = this.reuseApiContext && this.previousApi - ? [ - this.previousApi.operation, - ...candidates.filter((operation) => operation.key !== this.previousApi?.operation.key), - ] - : candidates; - this.apiOptions = operations.slice(0, MAX_API_OPTIONS).map((operation) => ({ - ref: this.refs.issue("api", operation, operation.key), - label: `${operation.method} ${operation.path} — ${operation.label}`, - operation, - })); - if (!this.apiOptions.length) { - this.stage = "clarify"; - this.observation = { noApiMatch: true }; - return; - } - this.stage = "resolve"; - this.observation = { - verifiedOperations: this.apiOptions.map(({ ref, operation }) => ({ - ref, - method: operation.method, - path: operation.path, - summary: operation.summary || operation.label, - description: operation.description, - parameters: operation.parameters, - response: { - type: operation.response.type, - description: operation.response.description, - fields: operation.response.fields - .slice(0, 12) - .map(({ name, type, description }) => ({ name, type, description })), - }, - })), - }; - } - - /** @param {(status: string) => void} onStatus @param {AbortSignal} signal */ - async tryDirectApi(onStatus, signal) { - const operation = this.apiOptions[0]?.operation; - if (!operation) return undefined; - const required = operation.parameters.filter((parameter) => parameter.required); - const values = literalArguments(this.request); - if (!values.length) { - const arguments_ = reusableArguments(operation, this.previousApi); - if (arguments_ === undefined && required.length) return undefined; - return await this.callApi( - operation, - arguments_ ?? {}, - onStatus, - signal, - ); - } - if (values.length !== required.length) return undefined; - const arguments_ = Object.fromEntries( - required.map((parameter, index) => [parameter.name, values[index]]), - ); - if ( - isExplicitComparison(this.request) && - this.previousApi?.operation.key === operation.key && - required.every((parameter) => - Object.hasOwn(this.previousApi?.arguments ?? {}, parameter.name) - ) - ) { - onStatus("Comparing API data…"); - const [previous, current] = await Promise.all([ - executeApi(operation, this.previousApi.arguments, signal), - executeApi(operation, arguments_, signal), - ]); - this.previousApi = { operation, arguments: current.arguments }; - return { - done: true, - apiGroundings: [previous, current].map((result) => ({ - question: this.request, - queries: this.queries, - ...result, - })), - }; - } - return await this.callApi(operation, arguments_, onStatus, signal); - } - - /** - * @param {import("./api/index.js").ApiOperation} operation - * @param {Record} arguments_ - * @param {(status: string) => void} onStatus - * @param {AbortSignal} signal - */ - async callApi(operation, arguments_, onStatus, signal) { - onStatus("Reading API…"); - const result = await executeApi(operation, arguments_, signal); - this.previousMetrics = []; - this.previousTopics = []; - this.previousApi = { operation, arguments: result.arguments }; - return { - done: true, - apiGrounding: { - question: this.request, - queries: this.queries, - ...result, - }, - }; - } - - /** @param {string[]} queries @param {(status: string) => void} onStatus */ - async rewrite(queries, onStatus) { - this.rewriteChanged = queries.length !== this.queries.length || - queries.some((query, index) => normalize(query) !== normalize(this.queries[index] ?? "")); - this.queries = queries; - this.query = queries.join(" / "); - this.rewritten = true; - return await this.search(onStatus) ?? { done: false }; - } - - async prepareMetricOptions() { - /** @type {MetricOption[]} */ - const metrics = []; - - for (const option of this.options) { - const kind = this.refs.kind(option.ref); - if (kind === "metric") { - const metric = this.refs.get(option.ref, "metric"); - metrics.push({ ref: option.ref, label: metric.label ?? label(metric.name), metric }); - } else if (kind === "guide") { - const guide = this.refs.get(option.ref, "guide"); - for (const series of guide.series) { - const metric = await metricByName(series.name); - if (!metric) continue; - const value = { ...metric, label: series.label }; - const ref = this.refs.issue("metric", value, metric.path); - metrics.push({ ref, label: series.label, metric: value }); - } - } - } - - this.metricOptions = uniqueMetricOptions(metrics).slice(0, MAX_OPTIONS); - if (!this.metricOptions.length) { - this.stage = "clarify"; - this.observation = { noMetric: true }; - return; - } - this.observation = { - verifiedMetrics: this.metricOptions.map(({ ref, label: metricLabel, metric }) => ({ - ref, - label: metricLabel, - path: metric.path, - unit: metric.suggestedUnit, - })), - }; - } - - /** @param {string[]} selected */ - async inspect(selected) { - const evidence = { - facts: /** @type {string[]} */ ([]), - sources: /** @type {{ revision: string, path: string, startLine: number, endLine?: number }[]} */ ([]), - excerpts: /** @type {{ revision: string, path: string, startLine: number, endLine?: number, content: string }[]} */ ([]), - }; - /** @type {string[]} */ - const topics = []; - /** @type {any[]} */ - const rememberedMetrics = []; - - for (const ref of selected) { - const kind = this.refs.kind(ref); - if (kind === "fact") { - const fact = this.refs.get(ref, "fact"); - topics.push(label(fact.fact.metric)); - const metric = await metricByName(fact.fact.metric); - if (metric) rememberedMetrics.push(metric); - evidence.facts.push(fact.answer); - evidence.sources.push(formulaCitation(fact)); - } else if (kind === "source") { - const match = this.refs.get(ref, "source"); - const excerpt = await this.source.read(match.path, match.startLine, match.endLine); - const focus = sourceFocus(this.request, [excerpt]); - evidence.excerpts.push({ ...excerpt, ...(focus ? { focus } : {}) }); - } else if (kind === "guide") { - const guide = this.refs.get(ref, "guide"); - if (guide.description) evidence.facts.push(guide.description); - topics.push(...guide.series.map((/** @type {any} */ series) => label(series.name))); - for (const series of guide.series) { - const metric = await metricByName(series.name); - if (metric) rememberedMetrics.push(metric); - } - } else if (kind === "metric") { - const metric = this.refs.get(ref, "metric"); - topics.push(label(metric.name)); - rememberedMetrics.push(metric); - const variants = this.focus === "variants" - ? await metricVariants(metric, this.query) - : undefined; - if (variants) { - const groups = variants.groups - .map((group) => `${group.family}: ${group.examples.join(", ")}`) - .join("; "); - evidence.facts.push( - `${label(metric.name)} has ${variants.totalSeries} available series. Cohorts: ${groups}.`, - ); - } - } - } - - if (this.formula && selected.some((ref) => this.refs.kind(ref) === "metric")) { - evidence.facts.unshift(this.formula.answer); - evidence.sources.push(formulaCitation(this.formula)); - } - - if (rememberedMetrics.length) { - this.rememberMetricValues(rememberedMetrics); - } else { - this.previousMetrics = []; - this.previousTopics = [...new Set(topics.length ? topics : this.queries)].slice(0, 4); - } - return { - done: true, - output: renderEvidence(evidence), - ...(evidence.excerpts.length - ? { - grounding: { - question: this.request, - excerpts: evidence.excerpts, - }, - } - : {}), - }; - } - - /** @param {Record} action */ - async read(action) { - const selected = uniqueRefs(action.refs); - const block = this.request.match(/\b(?:block\s*)?(\d{4,})\b/i)?.[1]; - const date = this.request.match(/\b\d{4}-\d{2}-\d{2}\b/)?.[0]; - const request = action.mode === "at" && action.at === undefined - ? { ...action, at: block ?? date } - : action; - const results = await Promise.all( - selected.map((ref) => readMetric(this.refs.get(ref, "metric"), request)), - ); - return { done: true, output: renderData(results) }; - } - - /** @param {Record} action */ - buildChart(action) { - const selected = uniqueRefs(action.refs); - const chosen = selected.map((ref) => this.refs.get(ref, "metric")); - const knownMetrics = new Map( - [...this.previousMetrics, ...chosen].map((metric) => [metric.path, metric]), - ); - const existingChart = this.outcome === "edit_existing_chart" - ? this.activeChart - : undefined; - const operation = typeof action.operation === "string" ? action.operation : "add"; - const added = chosen.map((/** @type {any} */ metric) => ({ - path: metric.path, - label: metric.label ?? label(metric.name), - })); - const prior = existingChart?.chart.series ?? []; - const paths = new Set(added.map(({ path }) => path)); - const series = !existingChart || operation === "replace" - ? added - : operation === "remove" - ? prior.filter((item) => !paths.has(item.path)) - : [...prior, ...added.filter((item) => !prior.some((old) => old.path === item.path))]; - if (!series.length) throw new Error("A chart needs at least one series"); - - const { unit, conflicts } = resolveChartUnit( - chosen, - existingChart?.chart.unit, - operation, - ); - if (conflicts.length) { - return { - done: true, - output: `Those metrics use different units (${conflicts.map((value) => value === "number" ? "**unitless**" : `**${value.toUpperCase()}**`).join(" and ")}), so they need separate charts. Which one should I chart?`, - artifacts: [], - }; - } - const artifact = createChartArtifact({ - title: typeof action.title === "string" && action.title.trim() - ? action.title.trim() - : series.map((item) => item.label).join(" and "), - unit, - series, - }); - this.activeChart = artifact; - this.rememberMetricValues( - artifact.chart.series.map((item) => knownMetrics.get(item.path)).filter(Boolean), - ); - return { - done: true, - output: `${existingChart ? "Updated" : "Built"} **${artifact.chart.title}** with ${artifact.chart.series.map((item) => item.label).join(", ")}.`, - artifacts: [artifact], - }; - } - - /** @param {{ scale?: string, view?: string }} action */ - restyleChart(action) { - if (!this.activeChart) throw new Error("There is no chart to update"); - const current = this.activeChart.chart; - const artifact = createChartArtifact({ - title: current.title, - unit: current.unit, - view: action.view ?? current.view, - scale: action.scale ?? current.scale, - series: current.series, - }); - this.activeChart = artifact; - const changes = [ - action.view ? `${action.view} view` : "", - action.scale ? `${action.scale} scale` : "", - ].filter(Boolean).join(" and "); - return { - done: true, - output: `Updated **${artifact.chart.title}** to use ${changes}.`, - artifacts: [artifact], - }; - } - - /** - * @param {Record} action - * @param {(status: string) => void} onStatus - */ - /** @param {Record} action @param {(status: string) => void} onStatus @param {AbortSignal} signal */ - async execute(action, onStatus, signal) { - const name = requiredString(action.action, "action"); - if (name === "clarify") { - const text = typeof action.text === "string" ? action.text.trim() : ""; - return { - done: true, - output: text || "I couldn't find a matching series. Which metric should I use instead?", - }; - } - if (this.stage === "search") { - if (name !== "search") throw new Error("The AI chose an invalid search action"); - this.outcome = requiredString(action.outcome, "outcome"); - if (this.outcome === "clarify_request") { - return { - done: true, - output: requiredString(action.clarification, "clarification"), - }; - } - if (this.outcome === "answer_general") { - return { done: true, general: true }; - } - const context = typeof action.context === "string" && action.context.trim() - ? action.context.trim() - : (this.previousTopics.length || this.previousApi) && referencesPrevious(this.request) - ? "reuse_previous" - : "new_topic"; - const proposed = Array.isArray(action.queries) && action.queries.length - ? requiredStrings(action.queries, "queries") - : [this.request]; - const cardinality = typeof action.cardinality === "string" - ? action.cardinality - : "single"; - const effectiveContext = context; - if ( - effectiveContext !== "new_topic" && - !this.previousTopics.length && - !this.previousApi - ) { - throw new Error("There is no previous verified topic to reuse"); - } - const previousTopics = [...this.previousTopics]; - const previousMetrics = [...this.previousMetrics]; - const routedQueries = this.outcome === "read_api" - ? proposed - : effectiveContext === "reuse_previous" - ? previousTopics - : effectiveContext === "extend_previous" - ? [...new Set([...previousTopics, ...proposed])] - : proposed; - this.contextMetrics = effectiveContext === "new_topic" ? [] : previousMetrics; - this.reuseApiContext = effectiveContext !== "new_topic" && Boolean(this.previousApi); - if (effectiveContext === "new_topic") { - this.previousMetrics = []; - this.previousTopics = []; - this.previousApi = undefined; - } - this.focus = evidenceFocus(this.request, "definition"); - this.comparison = cardinality === "multiple" || isExplicitComparison(this.request); - this.queries = this.comparison - ? completeComparisonQueries(routedQueries) - : routedQueries.slice(0, 1); - this.query = this.queries.join(" / "); - const searched = await this.search(onStatus); - if (this.outcome === "read_api" && this.apiOptions.length) { - const direct = await this.tryDirectApi(onStatus, signal); - if (direct) return direct; - } - return searched ?? { done: false }; - } - if (this.stage === "resolve" && this.outcome === "explain_from_verified_facts") { - if (name !== "answer") throw new Error("The AI chose an invalid evidence action"); - onStatus("Inspecting results…"); - return this.inspect(uniqueRefs(action.refs)); - } - if (this.stage === "rewrite") { - if (name !== "rewrite") throw new Error("The AI chose an invalid rewrite action"); - onStatus("Refining search…"); - return this.rewrite(requiredStrings(action.queries, "queries"), onStatus); - } - if (this.stage === "resolve" && this.outcome === "read_requested_value") { - if (name !== "read_data") throw new Error("The AI chose an invalid data action"); - this.rememberMetrics(uniqueRefs(action.refs)); - onStatus("Reading data…"); - return this.read(action); - } - if (this.stage === "resolve" && this.outcome === "read_api") { - if (name !== "call_api") throw new Error("The AI chose an invalid API action"); - const ref = requiredString(action.ref, "ref"); - const operation = this.refs.get(ref, "api"); - const supplied = action.arguments && typeof action.arguments === "object" - ? /** @type {Record} */ (action.arguments) - : {}; - const previousApi = this.previousApi; - const arguments_ = previousApi && previousApi.operation.key === operation.key - ? { ...previousApi.arguments, ...supplied } - : supplied; - return await this.callApi(operation, arguments_, onStatus, signal); - } - if (this.stage === "resolve") { - if (name !== "build_chart" && name !== "edit_chart") { - throw new Error("The AI chose an invalid chart action"); - } - onStatus("Building chart…"); - return this.buildChart(action); - } - if (name !== "clarify") throw new Error("The AI chose an invalid clarification action"); - return { done: true, output: requiredString(action.text, "text") }; - } - - /** @param {string[]} refs */ - rememberMetrics(refs) { - this.rememberMetricValues(refs.map((ref) => this.refs.get(ref, "metric"))); - } - - /** @param {any[]} metrics */ - rememberMetricValues(metrics) { - this.previousApi = undefined; - this.previousMetrics = [...new Map( - metrics.map((metric) => [metric.path, metric]), - ).values()].slice(0, 6); - this.previousTopics = this.previousMetrics.map((metric) => label(metric.name)); - this.recentMetrics = [...new Map( - [...this.previousMetrics, ...this.recentMetrics] - .map((metric) => [metric.path, metric]), - ).values()].slice(0, 6); - } - - metricPaths() { - return this.previousMetrics.map((metric) => metric.path); - } - - apiContext() { - return this.previousApi - ? { - key: this.previousApi.operation.key, - arguments: this.previousApi.arguments, - } - : undefined; - } -} diff --git a/website_next/ask/tools/session/capabilities.js b/website_next/ask/tools/session/capabilities.js new file mode 100644 index 000000000..c3fb04210 --- /dev/null +++ b/website_next/ask/tools/session/capabilities.js @@ -0,0 +1,540 @@ +import { normalize, tokenAffinity } from "../text.js"; + +/** @param {string} name @param {string} description @param {Record} properties @param {string[]} required */ +function tool(name, description, properties = {}, required = []) { + return { + type: "function", + function: { + name, + description, + parameters: { + type: "object", + properties, + required, + additionalProperties: false, + }, + }, + }; +} + +/** @param {{ ref: string }[]} options @param {number} [maxItems] @param {string} [description] */ +function references(options, maxItems = 4, description) { + return { + type: "array", + minItems: 1, + maxItems, + ...(description ? { description } : {}), + items: { + type: "string", + enum: options.map(({ ref }) => ref), + }, + }; +} + +/** @param {import("../api/index.js").ApiOperation[]} operations */ +function apiArguments(operations) { + const parameters = new Map(); + for (const operation of operations) { + for (const parameter of operation.parameters) { + const current = parameters.get(parameter.name); + parameters.set(parameter.name, { + type: "string", + description: [ + current?.description, + `${parameter.in}${parameter.required ? ", required" : ""} for ${operation.path}${parameter.description ? `: ${parameter.description}` : ""}`, + ].filter(Boolean).join(" "), + }); + } + } + return Object.fromEntries(parameters); +} + +/** @param {any} evidence */ +export function availableActions(evidence) { + const actions = []; + const hasVariantSelection = evidence.context.metrics.length && + evidence.metricOptions.some( + (/** @type {any} */ { origin }) => origin === "variant", + ); + if (evidence.apiOptions.length) actions.push("call_api"); + if (evidence.context.chart) { + const activePaths = new Set( + evidence.context.chart.chart.series.map( + (/** @type {{ path: string }} */ { path }) => path, + ), + ); + if ( + evidence.metricOptions.some( + (/** @type {any} */ { metric }) => !activePaths.has(metric.path), + ) + ) { + actions.push("add_chart_series"); + } + if ( + evidence.metricOptions.some( + (/** @type {any} */ { metric }) => activePaths.has(metric.path), + ) + ) { + actions.push("remove_chart_series"); + } + if (evidence.metricOptions.length) actions.push("replace_chart_series"); + } + if (evidence.context.chart) actions.push("set_chart_view_scale"); + if (hasVariantSelection) actions.push("select_metric_variant"); + if (evidence.metricOptions.length) { + actions.push( + "read_latest_metric", + "read_metric_at", + "read_metric_range", + "build_metric_chart", + "list_metric_variants", + ); + } + if (evidence.guideOptions.length || evidence.metricOptions.length) { + actions.push("explain_evidence"); + } + if (!evidence.metricOptions.length) actions.push("find_chart_metrics"); + actions.push("search_source"); + actions.push("describe_capabilities", "answer_general", "clarify"); + return actions; +} + +/** @type {Record} */ +const ROUTE_DESCRIPTIONS = { + add_chart_series: "Choose when the request adds series to activeChart.", + remove_chart_series: "Choose when the request removes series from activeChart.", + replace_chart_series: "Choose when the request replaces activeChart's series.", + set_chart_view_scale: "Choose when the request changes activeChart's view or scale.", + read_latest_metric: "Choose when the requested result is the latest value of a metric.", + read_metric_at: "Choose when the requested result is a metric value at a stated block height, date, or position.", + read_metric_range: "Choose when the requested result is metric values across a stated range.", + build_metric_chart: "Choose when the requested result is a new metric chart or graph.", + list_metric_variants: "Choose only when the requested result is a list of available cohorts, groupings, or series variants.", + select_metric_variant: "Choose when the request selects one matched cohort or series variant without requesting a value or chart yet.", + explain_evidence: "Choose for a Bitview metric definition grounded in matched metric evidence.", + find_chart_metrics: "Choose when the user asks which chart metrics exist but no exact metric matched yet.", + search_source: "Choose for a question about BRK repository code, implementation, callers, or source structure.", + call_api: "Choose for a concrete blockchain record or resource when a generated operation can accept the supplied or contextual identifier.", + describe_capabilities: "Choose only when the user asks what this assistant can do.", + answer_general: "Choose for ordinary Bitcoin knowledge, conversation, or writing.", + clarify: "Choose only when missing information would materially change the result.", +}; + +/** @param {unknown} value */ +function terms(value) { + return new Set( + normalize(value).split(" ").filter((term) => term.length >= 3), + ); +} + +/** + * Resolve only terminology encoded in the capability identifiers themselves. + * Natural-language descriptions contain incidental words and remain model + * context rather than routing rules. + * + * @param {any} evidence + * @param {string} question + */ +export function directAction(evidence, question) { + const actions = availableActions(evidence).filter( + (action) => action !== "answer_general" && action !== "clarify", + ); + const owners = new Map(); + for (const action of actions) { + for (const term of terms(action)) { + const values = owners.get(term) ?? []; + values.push(action); + owners.set(term, values); + } + } + const matched = new Set(); + for (const queryTerm of terms(question)) { + for (const [actionTerm, actionsForTerm] of owners) { + if ( + actionsForTerm.length === 1 && + tokenAffinity(queryTerm, actionTerm) >= 0.75 + ) { + matched.add(actionsForTerm[0]); + } + } + } + if (matched.size === 1) return [...matched][0]; + + const variants = evidence.metricOptions.filter( + (/** @type {any} */ { origin }) => origin === "variant", + ); + return matched.size === 0 && variants.length === 1 + ? "select_metric_variant" + : undefined; +} + +export function generalCapabilities() { + return [ + "Explain Bitcoin concepts and Bitview metrics from verified evidence", + "Read blockchain records and metric values", + "Build and edit metric charts", + "Search the current BRK source code", + ]; +} + +/** @param {any} evidence @param {string} action */ +export function capabilityMetrics(evidence, action) { + const mentioned = evidence.metricOptions.filter( + (/** @type {any} */ { origin }) => origin === "mentioned", + ); + const contextual = evidence.metricOptions.filter( + (/** @type {any} */ { origin }) => origin === "context", + ); + const recent = evidence.metricOptions.filter( + (/** @type {any} */ { origin }) => origin === "recent", + ); + const variants = evidence.metricOptions.filter( + (/** @type {any} */ { origin }) => origin === "variant", + ); + const options = mentioned.length + ? [...new Map( + [...mentioned, ...contextual].map((option) => [ + option.metric.path, + option, + ]), + ).values()] + : variants.length + ? action === "build_metric_chart" && contextual.length + ? [...variants, ...contextual] + : variants + : action === "build_metric_chart" && contextual.length && recent.length + ? [...contextual, ...recent] + : contextual.length + ? contextual + : recent.length + ? recent + : evidence.metricOptions; + const activePaths = new Set( + evidence.context.chart?.chart.series.map( + (/** @type {{ path: string }} */ { path }) => path, + ) ?? [], + ); + if (action === "add_chart_series") { + return options.filter( + (/** @type {any} */ { metric }) => !activePaths.has(metric.path), + ); + } + if (action === "remove_chart_series") { + return options.filter( + (/** @type {any} */ { metric }) => activePaths.has(metric.path), + ); + } + return options; +} + +/** @param {any} evidence */ +export function routeTools(evidence) { + const actions = availableActions(evidence); + return [ + tool( + "choose_capability", + actions.map((action) => `${action}: ${ROUTE_DESCRIPTIONS[action]}`).join(" "), + { + capability: { + type: "string", + enum: actions, + }, + ...(evidence.apiOptions.length + ? { + apiRef: { + type: "string", + enum: evidence.apiOptions.map( + (/** @type {any} */ { ref }) => ref, + ), + description: "When capability is call_api, the one matching generated operation.", + }, + } + : {}), + sourceQuery: { + type: "string", + description: "Provide only when capability is search_source: one compact lexical code-search query using symbols and implementation terms from the request and verified context.", + }, + }, + ["capability"], + ), + ]; +} + +/** @param {any} evidence @param {string} action */ +export function actionTool(evidence, action) { + const { metricOptions, apiOptions, sourceOptions, guideOptions } = evidence; + + if ( + action === "add_chart_series" || + action === "remove_chart_series" || + action === "replace_chart_series" + ) { + const options = capabilityMetrics(evidence, action); + return tool( + action, + ROUTE_DESCRIPTIONS[action], + { refs: references(options, 6) }, + ["refs"], + ); + } + if (action === "set_chart_view_scale") { + return tool( + action, + "Change only the explicitly requested active-chart view or scale.", + { + styles: { + type: "array", + minItems: 1, + maxItems: 2, + items: { + type: "string", + enum: ["line", "area", "stacked", "bar", "dots", "linear", "log"], + }, + }, + }, + ["styles"], + ); + } + if ( + action === "read_latest_metric" || + action === "read_metric_at" || + action === "read_metric_range" + ) { + const indexes = [...new Set( + metricOptions.flatMap( + (/** @type {any} */ { metric }) => metric.indexes ?? [], + ), + )]; + return tool( + action, + ROUTE_DESCRIPTIONS[action], + { + refs: references( + capabilityMetrics(evidence, action), + 4, + "Only the metrics actually requested; omit negated alternatives.", + ), + excludedRefs: references( + capabilityMetrics(evidence, action), + 4, + "Metrics explicitly negated or excluded by the request. Never repeat these in refs.", + ), + ...(action === "read_metric_at" + ? { + at: { + type: "string", + description: "Exact block height, date, or position copied from the request.", + }, + } + : {}), + ...(action === "read_metric_range" + ? { + index: { + type: "string", + ...(indexes.length ? { enum: indexes } : {}), + }, + } + : {}), + ...(action === "read_metric_range" + ? { + start: { type: "string" }, + end: { type: "string" }, + points: { type: "integer", minimum: 1, maximum: 120 }, + } + : {}), + }, + [ + "refs", + ...(action === "read_metric_at" ? ["at"] : []), + ], + ); + } + if (action === "build_metric_chart") { + const options = capabilityMetrics(evidence, action); + const asksContextDecision = + options.some((/** @type {any} */ { origin }) => origin === "context") && + options.some((/** @type {any} */ { origin }) => origin === "mentioned"); + return tool( + action, + "Select only the metrics requested for the new chart.", + { + refs: references( + options, + 6, + "Only positively requested chart series; omit negated or excluded alternatives.", + ), + excludedRefs: references( + options, + 6, + "Chart series explicitly negated or excluded by the request. Never repeat these in refs.", + ), + ...(asksContextDecision + ? { + includeContext: { + type: "boolean", + description: "Resolve indirect references first. True when the complete requested series set includes the current metric together with newly named metrics; false when the new subject replaces it.", + }, + } + : {}), + }, + ["refs", ...(asksContextDecision ? ["includeContext"] : [])], + ); + } + if (action === "list_metric_variants") { + return tool( + action, + "Select the one metric whose source-derived variants were requested.", + { refs: references(metricOptions, 1) }, + ["refs"], + ); + } + if (action === "select_metric_variant") { + return tool( + action, + "Select the one matched source-derived metric variant requested.", + { refs: references(capabilityMetrics(evidence, action), 1) }, + ["refs"], + ); + } + if (action === "explain_evidence") { + const evidenceOptions = [...sourceOptions, ...guideOptions]; + return tool( + action, + "Select the smallest sufficient verified evidence for the answer.", + { + refs: references(evidenceOptions, 1), + ...(metricOptions.length + ? { metrics: references(metricOptions, 4) } + : {}), + }, + ["refs"], + ); + } + if (action === "search_source") { + return tool( + action, + "Search the current BRK source snapshot before answering.", + { + query: { + type: "string", + description: "One compact lexical code-search query containing useful symbols, identifiers, or implementation terms from the request and verified source context.", + }, + }, + ["query"], + ); + } + if (action === "find_chart_metrics") { + return tool( + action, + "Search the generated chart metric catalog.", + { + query: { + type: "string", + description: "One compact metric search query using subjects from the request and verified conversation context.", + }, + }, + ["query"], + ); + } + if (action === "call_api") { + return tool( + action, + "Select the one generated read-only operation that directly answers the request.", + { + ref: { + type: "string", + enum: apiOptions.map((/** @type {any} */ { ref }) => ref), + }, + }, + ["ref"], + ); + } + if (action === "answer_general") { + return tool( + action, + "Answer without claiming live Bitview data or repository evidence.", + { + answer: { + type: "string", + description: "A clear concise answer to the exact request.", + }, + }, + ["answer"], + ); + } + if (action === "describe_capabilities") { + return tool( + action, + "Describe the assistant's generated capabilities.", + ); + } + if (action === "clarify") { + return tool( + action, + "Ask one concise question because essential information is missing.", + { question: { type: "string" } }, + ["question"], + ); + } + throw new Error(`Unsupported capability: ${action}`); +} + +/** @param {import("../api/index.js").ApiOperation} operation */ +export function apiArgumentTool(operation) { + return tool( + "provide_api_arguments", + `Copy only arguments for ${operation.method} ${operation.path}. Omit values not supplied by the request or verified context.`, + { + arguments: { + type: "object", + properties: apiArguments([operation]), + additionalProperties: false, + }, + }, + ["arguments"], + ); +} + +export const ROUTE_INSTRUCTION = `Choose one capability for the newest request from verified context and matches. Treat context.activeCapability as the active tool mode: continue it for an elliptical follow-up unless the newest request clearly selects a different available output. +The requested output wins: edit an active chart with its edit/style capability; otherwise choose the matching chart, latest-value, historical-value, range, variant-list, or variant-selection capability. +Use call_api for a concrete blockchain resource or its contextual follow-up, explain_evidence for a metric definition, find_chart_metrics to discover real chart series when none matched yet, describe_capabilities only for a request about the assistant itself, and answer_general for ordinary Bitcoin knowledge or conversation. +Use search_source only when the request explicitly asks about BRK repository code, source location, implementation, or callers. Never choose it merely because source matches exist. +Use clarify only when essential information is missing. With call_api select apiRef. With search_source provide sourceQuery. +Call choose_capability exactly once.`; + +/** @param {string} action */ +export function actionInstruction(action) { + const common = `Call ${action} exactly once. Copy selected refs and explicit values exactly. Never invent evidence, values, or arguments. Put only positively requested subjects in refs and explicitly rejected subjects in excludedRefs when that field is available; never put the same ref in both. Dependent follow-ups prefer context-origin subjects; newly named subjects prefer mentioned-origin matches. Similar candidates are alternatives, not a reason to select all of them.`; + if (action === "answer_general") { + return "Answer the newest request naturally in at most 60 words. Resolve elliptical follow-ups from the provided context without repeating that context first. Every claim must apply specifically to Bitcoin, not generic blockchain systems or Bitview's product, datasets, or availability. A request for examples needs at least three distinct named things, not a restatement or a list of benefits. Never invent observations, quantities, live, current, or real-time values. Do not mention routing, candidates, schemas, or internal instructions."; + } + if (action === "read_metric_at") { + return `${common} Copy the requested historical position exactly into at.`; + } + if (action === "build_metric_chart") { + return `${common} Resolve indirect references against the current metric, then select the complete requested chart series set.`; + } + if ( + action === "add_chart_series" || + action === "remove_chart_series" || + action === "replace_chart_series" + ) { + return `${common} Preserve the active chart and apply exactly the requested series change.`; + } + if (action === "set_chart_view_scale") { + return `${common} Apply only the explicitly requested view or scale.`; + } + if (action === "explain_evidence") { + return `${common} Select the one excerpt that directly answers the request. For a metric definition, prefer its computation or formula over UI configuration, imports, aggregation, or downstream usage. Select matching metrics when the request is about a metric.`; + } + if (action === "search_source") { + return `${common} Produce one compact lexical code-search query, not an answer. Preserve relevant symbols and identifiers from verified source context.`; + } + if (action === "find_chart_metrics") { + return `${common} Produce one compact catalog query containing only the metric subjects requested or referenced in verified context.`; + } + if (action === "call_api") { + return `${common} Choose one directly matching operation. Copy identifiers exactly and never invent a required argument.`; + } + return common; +} diff --git a/website_next/ask/tools/session/context.js b/website_next/ask/tools/session/context.js new file mode 100644 index 000000000..609b04292 --- /dev/null +++ b/website_next/ask/tools/session/context.js @@ -0,0 +1,69 @@ +import { apiByKey } from "../api/index.js"; +import { metricsByPaths } from "../metrics/index.js"; + +/** @param {any} message */ +function chartFrom(message) { + return message?.artifacts?.findLast?.( + (/** @type {any} */ artifact) => artifact.type === "chart", + ); +} + +/** + * Only the latest assistant response owns conversational tool context. This + * prevents an unrelated old chart, API call, or source result from silently + * becoming active again. + * + * @param {import("../../storage.js").StoredMessage[]} history + * @param {() => void} onProgress + */ +export async function loadSessionContext(history, onProgress) { + const assistantMessages = [...history].reverse().filter( + ({ role }) => role === "assistant", + ); + const message = assistantMessages[0]; + if (!message) { + return { + metrics: [], + recentMetrics: [], + source: [], + }; + } + + const chart = chartFrom(message); + const activePaths = [...new Set([ + ...(message.metricPaths ?? []), + ...(chart?.chart.series.map((/** @type {any} */ series) => series.path) ?? []), + ])]; + const recentPaths = [...new Set( + assistantMessages.slice(1, 4).flatMap((recent) => [ + ...(recent.metricPaths ?? []), + ...(chartFrom(recent)?.chart.series.map( + (/** @type {any} */ series) => series.path, + ) ?? []), + ]), + )].filter((path) => !activePaths.includes(path)); + const paths = [...activePaths, ...recentPaths]; + const [metrics, operation] = await Promise.all([ + paths.length ? metricsByPaths(paths, onProgress) : [], + message.apiContext?.key ? apiByKey(message.apiContext.key) : undefined, + ]); + + return { + ...(chart ? { chart } : {}), + metrics: metrics.filter(({ path }) => activePaths.includes(path)), + recentMetrics: metrics.filter(({ path }) => recentPaths.includes(path)), + ...(operation + ? { + api: { + operation, + arguments: message.apiContext?.arguments ?? {}, + fields: message.apiContext?.fields ?? [], + }, + } + : {}), + source: message.sourceContext ?? [], + ...(message.knowledgeContext + ? { knowledge: message.knowledgeContext } + : {}), + }; +} diff --git a/website_next/ask/tools/session/evidence.js b/website_next/ask/tools/session/evidence.js new file mode 100644 index 000000000..499a4be96 --- /dev/null +++ b/website_next/ask/tools/session/evidence.js @@ -0,0 +1,307 @@ +import { searchApi } from "../api/index.js"; +import { searchLearn } from "../learn.js"; +import { + mentionedMetricNames, + metricByName, + metricVariants, + searchMetrics, +} from "../metrics/index.js"; +import { normalize } from "../text.js"; +import { + explicitArguments, + hasRequiredArguments, +} from "../api/routing.js"; + +const MAX_METRICS = 5; +const MAX_API = 4; +const MAX_SOURCE = 6; +const MAX_GUIDES = 2; + +/** @param {string} value */ +function label(value) { + return value.replaceAll("_", " "); +} + +/** @param {any[]} values @param {(value: any) => string} key */ +function unique(values, key) { + const seen = new Set(); + return values.filter((value) => { + const itemKey = key(value); + if (seen.has(itemKey)) return false; + seen.add(itemKey); + return true; + }); +} + +/** @param {any} metric */ +function acceptsMetric(metric) { + return Number(metric.matchedTerms ?? 0) >= 2; +} + +/** @param {any} operation @param {string} question */ +function acceptsApi(operation, question) { + const specificity = Number(operation.specificity ?? 0); + const required = operation.parameters.some( + (/** @type {any} */ parameter) => parameter.required, + ); + if (!required) { + return Number(operation.titleMatchedTerms ?? 0) > 0 && + specificity >= 2.5; + } + return specificity >= 1.5 && + hasRequiredArguments(operation, explicitArguments(operation, question)); +} + +/** @param {any} match */ +function acceptsSource(match) { + return Number(match.matched ?? 0) >= + Math.min(2, Number(match.queryTerms ?? 0)); +} + +/** @param {any} guide */ +function acceptsGuide(guide) { + return Number(guide.titleCoverage ?? 0) >= 0.75; +} + +/** @param {any} metric */ +async function sourceMetricSubject(metric) { + const variants = await metricVariants(metric, ""); + const names = variants?.series.map((/** @type {any} */ series) => + normalize(series.name).split(" ") + ) ?? []; + if (names.length < 2) return label(metric.name); + + const shortest = Math.min(...names.map((name) => name.length)); + const suffix = []; + for (let offset = 1; offset <= shortest; offset += 1) { + const word = names[0].at(-offset); + if (!word || names.some((name) => name.at(-offset) !== word)) break; + suffix.unshift(word); + } + return suffix.length >= 2 ? suffix.join(" ") : label(metric.name); +} + +/** + * Search generated metric, API, and guide catalogs before asking the model + * what the request means. Repository excerpts are loaded only if selected. + * + * @param {Object} options + * @param {string} options.question + * @param {any} options.context + * @param {import("../refs.js").AskRefs} options.refs + * @param {(status: string) => void} options.onStatus + */ +export async function collectEvidence({ + question, + context, + refs, + onStatus, +}) { + const [foundMetrics, foundApi, searchedGuides, mentionedNames] = + await Promise.all([ + searchMetrics( + [question], + MAX_METRICS, + [], + () => onStatus("Indexing metrics…"), + ), + searchApi( + [question], + MAX_API, + () => onStatus("Indexing API…"), + ), + searchLearn(question, MAX_GUIDES), + mentionedMetricNames(question), + ]); + const foundGuides = searchedGuides.filter(acceptsGuide); + const mentionedMetrics = (await Promise.all( + mentionedNames.map((name) => metricByName(name)), + )).filter(Boolean); + const foundByPath = new Map( + foundMetrics.map((metric) => [metric.path, metric]), + ); + const normalizedQuestion = normalize(question); + const linkedMetrics = mentionedMetrics.map((/** @type {any} */ metric) => { + const match = foundByPath.get(metric.path); + const normalizedName = normalize(metric.name); + const explicit = normalizedQuestion === normalizedName || + normalizedName.split(" ").length > 1 || + Number(match?.matchedTerms ?? 0) >= 2; + return { + ...metric, + ...(match ?? {}), + origin: explicit ? "mentioned" : "search", + }; + }); + const variantMetrics = (await Promise.all( + context.metrics.map((/** @type {any} */ metric) => + metricVariants(metric, question) + ), + )).flatMap((variants) => + variants?.series + .filter((/** @type {any} */ { matchedTerms }) => matchedTerms > 0) + .map((/** @type {any} */ metric) => { + const name = label(metric.name); + const selector = label(metric.selector); + return { + ...metric, + label: name === selector || name.startsWith(`${selector} `) + ? name + : `${selector} ${name}`.trim(), + origin: "variant", + }; + }) ?? [] + ); + + const metrics = unique( + [ + ...linkedMetrics.filter(({ origin }) => origin === "mentioned"), + ...variantMetrics, + ...context.metrics.map((/** @type {any} */ metric) => ({ + ...metric, + origin: "context", + })), + ...context.recentMetrics.map((/** @type {any} */ metric) => ({ + ...metric, + origin: "recent", + })), + ...linkedMetrics.filter(({ origin }) => origin !== "mentioned"), + ...foundMetrics.filter(acceptsMetric), + ], + (metric) => metric.path, + ).slice(0, MAX_METRICS); + const api = unique( + [ + ...(context.api ? [context.api.operation] : []), + ...foundApi.filter((operation) => acceptsApi(operation, question)), + ], + (operation) => operation.key, + ).slice(0, MAX_API); + const guides = unique( + foundGuides, + (guide) => guide.breadcrumbs.join("/"), + ).slice(0, MAX_GUIDES); + + const metricOptions = metrics.map((metric) => ({ + ref: refs.issue( + "metric", + metric, + metric.path, + metric.label ?? label(metric.name), + ), + label: metric.label ?? label(metric.name), + metric, + origin: metric.origin ?? "search", + })); + const apiOptions = api.map((operation) => ({ + ref: refs.issue("api", operation, operation.key, operation.label), + label: operation.label, + operation, + })); + const sourceOptions = context.source.map((/** @type {any} */ match) => ({ + ref: refs.issue( + "source", + match, + `${match.revision}:${match.path}:${match.startLine}`, + match.path.split("/").at(-1) ?? match.path, + ), + label: `${match.path}:${match.startLine}`, + source: match, + })); + const guideOptions = guides.map((guide) => ({ + ref: refs.issue( + "guide", + guide, + guide.breadcrumbs.join("/"), + guide.title, + ), + label: guide.title, + guide, + })); + + return { + metricOptions, + apiOptions, + sourceOptions, + guideOptions, + context, + }; +} + +/** + * Source retrieval is intentionally lazy. Routing, API reads, metric values, + * and chart edits do not need repository excerpts. + * + * @param {Object} options + * @param {string} options.question + * @param {any} options.evidence + * @param {import("../source/index.js").AskSource} options.source + * @param {import("../refs.js").AskRefs} options.refs + * @param {(status: string) => void} options.onStatus + */ +export async function collectSourceOptions({ + question, + evidence, + source, + refs, + onStatus, +}) { + const metric = evidence.metricOptions[0]?.metric; + const metricSubject = metric + ? await sourceMetricSubject(metric) + : undefined; + const queries = [ + ...(metricSubject + ? [{ + query: metricSubject, + focus: /** @type {const} */ ("definition"), + }, { + query: metricSubject, + focus: /** @type {const} */ ("implementation"), + }] + : []), + { query: question, focus: undefined }, + ].filter((value, index, values) => + values.findIndex((candidate) => + candidate.query === value.query && candidate.focus === value.focus + ) === index + ); + onStatus("Searching source…"); + const results = await Promise.all( + queries.map(({ query, focus }) => + source.search( + query, + undefined, + focus, + ({ loaded, total }) => + onStatus(`Indexing source · ${loaded} / ${total}`), + ) + ), + ); + const sources = unique( + [ + ...results.flatMap((result) => + result.matches + .filter(acceptsSource) + .map((/** @type {any} */ match) => ({ + ...match, + revision: result.revision, + })) + ).sort((left, right) => + Number(right.score ?? 0) - Number(left.score ?? 0) + ), + ...evidence.context.source, + ], + (match) => `${match.revision}:${match.path}:${match.startLine}`, + ).slice(0, MAX_SOURCE); + return sources.map((match) => ({ + ref: refs.issue( + "source", + match, + `${match.revision}:${match.path}:${match.startLine}`, + match.path.split("/").at(-1) ?? match.path, + ), + label: `${match.path}:${match.startLine}`, + source: match, + })); +} diff --git a/website_next/ask/tools/session/executor.js b/website_next/ask/tools/session/executor.js new file mode 100644 index 000000000..a4a15d54b --- /dev/null +++ b/website_next/ask/tools/session/executor.js @@ -0,0 +1,546 @@ +import { executeApi } from "../api/execute.js"; +import { reusableArguments } from "../api/routing.js"; +import { createChartArtifact } from "../chart.js"; +import { resolveChartUnit } from "../chart/units.js"; +import { readMetric } from "../data.js"; +import { metricVariants, searchMetrics } from "../metrics/index.js"; +import { renderData } from "../render.js"; + +const CHART_VIEWS = new Set(["line", "area", "stacked", "bar", "dots"]); +const CHART_SCALES = new Set(["linear", "log"]); + +/** @param {unknown} value @param {string} name */ +function requiredString(value, name) { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`The AI did not provide ${name}`); + } + return value.trim(); +} + +/** @param {unknown} value */ +function uniqueRefs(value) { + if ( + !Array.isArray(value) || + !value.length || + value.some((ref) => typeof ref !== "string") + ) { + throw new Error("The AI did not select verified evidence"); + } + return [...new Set(value)]; +} + +/** @param {string} value */ +function label(value) { + return value.replaceAll("_", " "); +} + +/** @param {string | undefined} value */ +function inlineCode(value) { + if (!value) return []; + return value + .split("`") + .filter((_, index) => index % 2 === 1) + .map((term) => term.trim()) + .filter(Boolean); +} + +/** @param {string | undefined} value */ +function sourceSubject(value) { + return inlineCode(value).find((term) => + !term.includes("/") && !term.includes("\\") + ); +} + +export class CapabilityExecutor { + /** + * @param {Object} options + * @param {string} options.question + * @param {any} options.evidence + * @param {import("../refs.js").AskRefs} options.refs + * @param {import("../source/index.js").AskSource} options.source + */ + constructor({ question, evidence, refs, source }) { + this.question = question; + this.evidence = evidence; + this.refs = refs; + this.source = source; + } + + /** @param {unknown} value @param {"metric" | "source" | "guide"} kind */ + selected(value, kind) { + return uniqueRefs(value).map((ref) => ({ + ref, + value: this.refs.get(ref, kind), + })); + } + + /** @param {Record} arguments_ */ + answerGeneral(arguments_) { + const answer = requiredString(arguments_.answer, "an answer"); + return { + done: true, + output: answer, + knowledgeContext: { + title: this.question.slice(0, 160), + description: answer, + }, + }; + } + + /** @param {Record} arguments_ */ + clarify(arguments_) { + return { + done: true, + output: requiredString(arguments_.question, "a clarification question"), + }; + } + + /** @param {Record} arguments_ */ + describeCapabilities(arguments_) { + const capabilities = Array.isArray(arguments_.capabilities) + ? arguments_.capabilities.filter((value) => typeof value === "string") + : []; + if (!capabilities.length) { + throw new Error("The assistant capabilities are unavailable"); + } + return { + done: true, + output: `I can:\n\n${capabilities.map((value) => `- ${value}`).join("\n")}`, + }; + } + + /** @param {Record} arguments_ */ + async explain(arguments_) { + const selected = [ + ...uniqueRefs(arguments_.refs), + ...(Array.isArray(arguments_.metrics) + ? uniqueRefs(arguments_.metrics) + : []), + ]; + const sources = []; + const metrics = []; + const guides = []; + + for (const ref of selected) { + const kind = this.refs.kind(ref); + if (kind === "source") sources.push(this.refs.get(ref, "source")); + else if (kind === "metric") metrics.push(this.refs.get(ref, "metric")); + else if (kind === "guide") guides.push(this.refs.get(ref, "guide")); + else throw new Error(`${ref} cannot ground an explanation`); + } + + return { + done: true, + metricPaths: metrics.map((metric) => metric.path), + sourceContext: sources, + grounding: { + question: this.question, + metrics: metrics.map((metric) => ({ + name: label(metric.name), + path: metric.path, + unit: metric.suggestedUnit, + })), + facts: guides.map((guide) => guide.description).filter(Boolean), + excerpts: sources, + }, + }; + } + + /** @param {Record} arguments_ @param {(status: string) => void} onStatus */ + async searchSource(arguments_, onStatus) { + const query = requiredString(arguments_.query, "a source search query"); + const subject = sourceSubject( + this.evidence.context.knowledge?.description, + ); + const context = this.evidence.context.source; + const subjectContext = subject + ? context.filter((/** @type {{ content: string }} */ source) => + source.content.includes(subject) + ) + : []; + const scopedContext = subjectContext.length ? subjectContext : context; + const paths = [...new Set(scopedContext.map( + (/** @type {{ path: string }} */ { path }) => path, + ))].slice(0, 2); + const contextualQuery = subject + ? `${query} ${subject}` + : query; + const searches = [ + { query, path: undefined }, + ...(contextualQuery === query + ? [] + : [{ query: contextualQuery, path: undefined }]), + ...(subject ? [{ query: subject, path: undefined }] : []), + ...paths.map((path) => ({ query: contextualQuery, path })), + ]; + + onStatus("Searching source…"); + const results = await Promise.all( + searches.map(({ query: scopedQuery, path }) => + this.source.search( + scopedQuery, + path, + undefined, + ({ loaded, total }) => + onStatus(`Indexing source · ${loaded} / ${total}`), + ) + ), + ); + const scopedResults = results.filter((_, index) => searches[index].path); + const contextualResult = contextualQuery === query + ? undefined + : results[1]; + const subjectResult = subject + ? results[contextualQuery === query ? 1 : 2] + : undefined; + const rawResult = results[0]; + const seeded = this.evidence.sourceOptions.map( + (/** @type {{ source: any }} */ { source }) => source, + ); + const excerpts = [...new Map([ + ...(paths.length ? [] : seeded), + ...scopedResults.flatMap((result) => + result.matches.slice(0, 1).map((/** @type {any} */ match) => ({ + ...match, + revision: result.revision, + })) + ), + ...(subjectResult?.matches.slice(0, 3).map( + (/** @type {any} */ match) => ({ + ...match, + revision: subjectResult.revision, + }), + ) ?? []), + ...(contextualResult?.matches.slice(0, 3).map( + (/** @type {any} */ match) => ({ + ...match, + revision: contextualResult.revision, + }), + ) ?? []), + ...(paths.length ? seeded : []), + ...rawResult.matches.slice(0, 3).map((/** @type {any} */ match) => ({ + ...match, + revision: rawResult.revision, + })), + ].map((excerpt) => [ + `${excerpt.revision}:${excerpt.path}:${excerpt.startLine}`, + excerpt, + ])).values()].slice(0, 3); + if (!excerpts.length) { + return { + done: true, + output: "I could not find enough verified source evidence to answer that.", + }; + } + return { + done: true, + sourceContext: excerpts, + grounding: { + question: this.question, + metrics: [], + facts: [], + excerpts, + }, + }; + } + + /** @param {Record} arguments_ */ + async listVariants(arguments_) { + const [{ value: metric }] = this.selected(arguments_.refs, "metric"); + if (metric.origin === "variant") { + return this.selectVariant(arguments_); + } + const variants = await metricVariants(metric, this.question); + if (!variants) { + return { + done: true, + output: `I found one **${label(metric.name)}** series and no cohort variants.`, + metricPaths: [metric.path], + }; + } + const groups = variants.groups + .map((group) => + group.examples.length === 1 && group.examples[0] === group.family + ? group.family + : `${group.family}: ${group.examples.join(", ")}` + ) + .join("; "); + return { + done: true, + output: `**${label(metric.name)}** has ${variants.totalSeries} available series. Variant groups: ${groups}.`, + metricPaths: [metric.path], + }; + } + + /** @param {Record} arguments_ @param {(status: string) => void} onStatus */ + async findChartMetrics(arguments_, onStatus) { + const query = requiredString(arguments_.query, "a metric search query"); + const context = this.evidence.context.knowledge; + const queries = [...new Set([ + query, + this.question, + context?.title, + context?.description, + ].filter((value) => typeof value === "string" && value.trim()))]; + onStatus("Searching metrics…"); + const metrics = (await searchMetrics(queries, 12)) + .filter(({ matchedTerms }) => Number(matchedTerms ?? 0) > 0) + .filter((metric, index, values) => + values.findIndex(({ name }) => name === metric.name) === index + ) + .slice(0, 6); + if (!metrics.length) { + return { + done: true, + output: "I could not find a chart metric matching that description.", + }; + } + return { + done: true, + output: `I found these chart metrics:\n\n${ + metrics.map((metric) => + `- **${label(metric.name)}**${ + metric.suggestedUnit ? ` · ${metric.suggestedUnit}` : "" + }` + ).join("\n") + }`, + metricPaths: metrics.map(({ path }) => path), + }; + } + + /** @param {Record} arguments_ */ + selectVariant(arguments_) { + const [{ value: metric }] = this.selected(arguments_.refs, "metric"); + return { + done: true, + output: `Selected **${metric.label ?? label(metric.name)}**.`, + metricPaths: [metric.path], + }; + } + + /** @param {Record} arguments_ @param {"latest" | "at" | "range"} mode */ + async read(arguments_, mode) { + const selected = this.selected(arguments_.refs, "metric"); + if (mode === "at") requiredString(arguments_.at, "a block height or date"); + const results = await Promise.all( + selected.map(({ value: metric }) => + readMetric(metric, { ...arguments_, mode }) + ), + ); + return { + done: true, + output: renderData(results), + metricPaths: selected.map(({ value }) => value.path), + }; + } + + /** @param {any[]} chosen @param {any} [existing] @param {"add" | "remove" | "replace"} [operation] */ + chart(chosen, existing, operation) { + const prior = /** @type {any[]} */ (existing?.chart.series ?? []); + const added = chosen.map((metric) => ({ + path: metric.path, + label: metric.label ?? label(metric.name), + })); + const selectedPaths = new Set(added.map(({ path }) => path)); + const series = !existing || operation === "replace" + ? added + : operation === "remove" + ? prior.filter(({ path }) => !selectedPaths.has(path)) + : [ + ...prior, + ...added.filter(({ path }) => + !prior.some((item) => item.path === path) + ), + ]; + if (!series.length) throw new Error("A chart needs at least one series"); + + const { unit, conflicts } = resolveChartUnit( + chosen, + existing?.chart.unit, + operation ?? "replace", + ); + if (conflicts.length) { + return { + done: true, + output: `Those metrics use different units (${conflicts.map((value) => value === "number" ? "**unitless**" : `**${value.toUpperCase()}**`).join(" and ")}), so they need separate charts. Which one should I chart?`, + artifacts: [], + }; + } + const artifact = createChartArtifact({ + title: series.map((item) => item.label).join(" and "), + unit, + view: existing?.chart.view, + scale: existing?.chart.scale, + series, + }); + const known = new Map( + [...this.evidence.context.metrics, ...chosen] + .map((metric) => [metric.path, metric]), + ); + return { + done: true, + output: `${existing ? "Updated" : "Built"} **${artifact.chart.title}** with ${artifact.chart.series.map((item) => item.label).join(", ")}.`, + artifacts: [artifact], + metricPaths: [...new Set([ + ...artifact.chart.series.map(({ path }) => path), + ...(operation === "remove" || operation === "replace" + ? chosen.map(({ path }) => path) + : []), + ])].filter((path) => known.has(path)), + }; + } + + /** @param {Record} arguments_ */ + buildChart(arguments_) { + const selected = this.selected(arguments_.refs, "metric") + .map(({ value }) => value); + const chosen = [...new Map([ + ...(arguments_.includeContext === true + ? this.evidence.context.metrics + : []), + ...selected, + ].map((metric) => [metric.path, metric])).values()]; + return this.chart(chosen); + } + + /** + * @param {Record} arguments_ + * @param {"add" | "remove" | "replace"} operation + */ + editChart(arguments_, operation) { + const active = this.evidence.context.chart; + if (!active) throw new Error("There is no active chart to update"); + const chosen = this.selected(arguments_.refs, "metric") + .map(({ value }) => value); + return this.chart(chosen, active, operation); + } + + /** @param {Record} arguments_ */ + styleChart(arguments_) { + const active = this.evidence.context.chart; + if (!active) throw new Error("There is no active chart to update"); + const styles = Array.isArray(arguments_.styles) + ? arguments_.styles.filter((style) => typeof style === "string") + : []; + const view = styles.find((style) => CHART_VIEWS.has(style)); + const scale = styles.find((style) => CHART_SCALES.has(style)); + if (!view && !scale) throw new Error("The AI did not select a chart style"); + const artifact = createChartArtifact({ + title: active.chart.title, + unit: active.chart.unit, + view: view ?? active.chart.view, + scale: scale ?? active.chart.scale, + series: active.chart.series, + }); + return { + done: true, + output: `Updated **${artifact.chart.title}** to use ${[ + view ? `${view} view` : "", + scale ? `${scale} scale` : "", + ].filter(Boolean).join(" and ")}.`, + artifacts: [artifact], + metricPaths: artifact.chart.series.map(({ path }) => path), + }; + } + + /** + * @param {Record} arguments_ + * @param {(status: string) => void} onStatus + * @param {AbortSignal} signal + */ + async callApi(arguments_, onStatus, signal) { + const ref = requiredString(arguments_.ref, "an API reference"); + const operation = this.refs.get(ref, "api"); + const supplied = arguments_.arguments && + typeof arguments_.arguments === "object" && + !Array.isArray(arguments_.arguments) + ? /** @type {Record} */ (arguments_.arguments) + : {}; + const reused = reusableArguments(operation, this.evidence.context.api); + onStatus("Reading API…"); + const result = await executeApi( + operation, + { ...(reused ?? {}), ...supplied }, + signal, + ); + return { + done: true, + apiContext: { + key: operation.key, + arguments: result.arguments, + }, + apiGrounding: { + question: this.question, + previousFields: this.evidence.context.api?.fields ?? [], + ...result, + }, + }; + } + + /** + * @param {{ name: string, arguments: Record }} call + * @param {(status: string) => void} onStatus + * @param {AbortSignal} signal + */ + async execute(call, onStatus, signal) { + signal.throwIfAborted(); + if (call.name === "answer_general") { + return this.answerGeneral(call.arguments); + } + if (call.name === "describe_capabilities") { + return this.describeCapabilities(call.arguments); + } + if (call.name === "clarify") return this.clarify(call.arguments); + if (call.name === "explain_evidence") { + return await this.explain(call.arguments); + } + if (call.name === "search_source") { + return await this.searchSource(call.arguments, onStatus); + } + if (call.name === "list_metric_variants") { + return await this.listVariants(call.arguments); + } + if (call.name === "find_chart_metrics") { + return await this.findChartMetrics(call.arguments, onStatus); + } + if (call.name === "select_metric_variant") { + return this.selectVariant(call.arguments); + } + if (call.name === "read_latest_metric") { + onStatus("Reading data…"); + return await this.read(call.arguments, "latest"); + } + if (call.name === "read_metric_at") { + onStatus("Reading data…"); + return await this.read(call.arguments, "at"); + } + if (call.name === "read_metric_range") { + onStatus("Reading data…"); + return await this.read(call.arguments, "range"); + } + if (call.name === "build_metric_chart") { + onStatus("Building chart…"); + return this.buildChart(call.arguments); + } + if (call.name === "add_chart_series") { + onStatus("Updating chart…"); + return this.editChart(call.arguments, "add"); + } + if (call.name === "remove_chart_series") { + onStatus("Updating chart…"); + return this.editChart(call.arguments, "remove"); + } + if (call.name === "replace_chart_series") { + onStatus("Updating chart…"); + return this.editChart(call.arguments, "replace"); + } + if (call.name === "set_chart_view_scale") { + onStatus("Updating chart…"); + return this.styleChart(call.arguments); + } + if (call.name === "call_api") { + return await this.callApi(call.arguments, onStatus, signal); + } + throw new Error(`Unsupported AI capability: ${call.name}`); + } +} diff --git a/website_next/ask/tools/session/index.js b/website_next/ask/tools/session/index.js new file mode 100644 index 000000000..984baaaaa --- /dev/null +++ b/website_next/ask/tools/session/index.js @@ -0,0 +1,570 @@ +import { AskRefs } from "../refs.js"; +import { + ROUTE_INSTRUCTION, + actionInstruction, + actionTool, + apiArgumentTool, + availableActions, + capabilityMetrics, + directAction, + generalCapabilities, + routeTools, +} from "./capabilities.js"; +import { loadSessionContext } from "./context.js"; +import { collectEvidence, collectSourceOptions } from "./evidence.js"; +import { CapabilityExecutor } from "./executor.js"; +import { normalize } from "../text.js"; +import { + explicitArguments, + hasRequiredArguments, + reusableArguments, + validatedArguments, +} from "../api/routing.js"; + +/** @param {unknown[]} values */ +function schemaTokens(values) { + return new Set( + normalize(values.filter(Boolean).join(" ")) + .split(" ") + .filter((token) => token.length >= 3), + ); +} + +/** @param {Set} query @param {Set} document */ +function overlaps(query, document) { + return [...query].some((token) => document.has(token)); +} + +export class AskToolSession { + /** @param {import("../source/index.js").AskSource} source */ + constructor(source) { + this.source = source; + } + + /** @param {string} question @param {import("../../storage.js").StoredMessage[]} history @param {(status: string) => void} onStatus */ + async begin(question, history, onStatus) { + const refs = new AskRefs(); + this.refs = refs; + const context = await loadSessionContext( + history, + () => onStatus("Indexing context…"), + ); + onStatus("Searching tools…"); + const evidence = await collectEvidence({ + question, + context, + refs, + onStatus, + }); + this.question = question; + this.evidence = evidence; + this.executor = new CapabilityExecutor({ + question, + evidence, + refs, + source: this.source, + }); + } + + question = ""; + /** @type {Awaited> | undefined} */ + evidence; + /** @type {CapabilityExecutor | undefined} */ + executor; + /** @type {AskRefs | undefined} */ + refs; + + routeTools() { + if (!this.evidence) throw new Error("Tool session is not ready"); + return routeTools(this.evidence); + } + + hasSourceContext() { + return Boolean(this.evidence?.context.source.length); + } + + routeMessages() { + if (!this.evidence) throw new Error("Tool session is not ready"); + const { context, metricOptions, apiOptions, sourceOptions, guideOptions } = + this.evidence; + return [ + { + role: /** @type {const} */ ("system"), + content: ROUTE_INSTRUCTION, + }, + { + role: /** @type {const} */ ("user"), + content: JSON.stringify({ + request: this.question, + context: { + activeCapability: context.chart + ? "chart" + : context.api + ? "api" + : context.metrics.length + ? "metric" + : context.source.length + ? "source" + : context.knowledge + ? "general" + : undefined, + ...(context.chart + ? { + activeChart: { + title: context.chart.chart.title, + series: context.chart.chart.series.map( + (/** @type {any} */ { path, label }) => ({ path, label }), + ), + }, + } + : {}), + ...(context.metrics.length + ? { + metrics: context.metrics.map( + (/** @type {any} */ metric) => ({ + name: metric.name, + path: metric.path, + }), + ), + } + : {}), + ...(context.recentMetrics.length + ? { + recentMetrics: context.recentMetrics.map( + (/** @type {any} */ metric) => ({ + name: metric.name, + path: metric.path, + }), + ), + } + : {}), + ...(context.api + ? { + api: { + operation: context.api.operation.label, + arguments: context.api.arguments, + }, + } + : {}), + ...(context.source.length + ? { + source: context.source.map( + (/** @type {any} */ source) => source.path, + ), + } + : {}), + ...(context.knowledge ? { concept: context.knowledge } : {}), + }, + matches: { + metrics: metricOptions.map(({ label, origin }) => ({ + label, + origin, + })), + api: apiOptions.map(({ label, operation }) => ({ + label, + required: operation.parameters + .filter((/** @type {any} */ parameter) => parameter.required) + .map((/** @type {any} */ parameter) => ({ + name: parameter.name, + type: parameter.valueType || parameter.type, + description: parameter.description, + })), + returns: operation.response.fields + .slice(0, 8) + .map((/** @type {any} */ field) => field.name), + })), + source: sourceOptions.map((/** @type {any} */ { ref, source }) => ({ + ref, + path: source.path, + })), + guides: guideOptions.map(({ label }) => label), + }, + }), + }, + ]; + } + + directRoute() { + if (!this.evidence) return undefined; + const query = schemaTokens([this.question]); + const contextKey = this.evidence.context.api?.operation.key; + for (const { ref, operation } of this.evidence.apiOptions) { + const required = schemaTokens( + operation.parameters + .filter((/** @type {any} */ parameter) => parameter.required) + .flatMap((/** @type {any} */ parameter) => [ + parameter.name, + parameter.description, + ]), + ); + const returned = schemaTokens( + operation.response.fields.flatMap((/** @type {any} */ field) => [ + field.name, + field.description, + ]), + ); + const fieldMatch = overlaps(query, returned); + const suppliedResource = required.size > 0 && overlaps(query, required); + if ( + fieldMatch && + (suppliedResource || operation.key === contextKey) + ) { + return { + action: "call_api", + call: { + name: "call_api", + arguments: { ref }, + }, + }; + } + } + const action = directAction(this.evidence, this.question); + if (action === "search_source") { + return { + action, + call: { + name: action, + arguments: { query: this.question }, + }, + }; + } + if (action) { + return { + action, + call: this.directCall(action), + }; + } + return undefined; + } + + /** @param {string} action @param {(status: string) => void} onStatus */ + async prepareAction(action, onStatus) { + if ( + action !== "explain_evidence" || + !this.evidence || + !this.refs + ) return; + this.evidence.sourceOptions = await collectSourceOptions({ + question: this.question, + evidence: this.evidence, + source: this.source, + refs: this.refs, + onStatus, + }); + } + + /** @param {string} action */ + actionTool(action) { + const evidence = this.evidence; + if (!evidence) throw new Error("Tool session is not ready"); + if (!availableActions(evidence).includes(action)) { + throw new Error(`Unavailable AI capability: ${action}`); + } + return actionTool(evidence, action); + } + + /** @param {string} action */ + directCall(action) { + const evidence = this.evidence; + if (!evidence) return undefined; + + const metrics = capabilityMetrics(evidence, action); + if (action === "describe_capabilities") { + return { + name: action, + arguments: { capabilities: generalCapabilities() }, + }; + } + if ( + action === "add_chart_series" || + action === "remove_chart_series" || + action === "replace_chart_series" + ) { + const activePaths = new Set( + evidence.context.chart?.chart.series.map( + (/** @type {{ path: string }} */ { path }) => path, + ) ?? [], + ); + const mentioned = evidence.metricOptions.filter( + ({ metric, origin }) => + origin === "mentioned" && + ( + action === "add_chart_series" + ? !activePaths.has(metric.path) + : activePaths.has(metric.path) + ), + ); + if (mentioned.length === 1) { + return { + name: action, + arguments: { refs: [mentioned[0].ref] }, + }; + } + } + if ( + ( + action === "read_latest_metric" || + action === "select_metric_variant" || + action === "build_metric_chart" || + action === "add_chart_series" || + action === "remove_chart_series" || + action === "replace_chart_series" + ) && + metrics.length === 1 + ) { + return { + name: action, + arguments: { refs: [metrics[0].ref] }, + }; + } + + if (action === "list_metric_variants") { + const variants = evidence.metricOptions.filter( + ({ origin }) => origin === "variant", + ); + if (variants.length === 1) { + return { + name: action, + arguments: { refs: [variants[0].ref] }, + }; + } + const bases = evidence.metricOptions.filter( + ({ origin }) => origin === "mentioned" || origin === "context", + ); + if (!variants.length && bases.length === 1) { + return { + name: action, + arguments: { refs: [bases[0].ref] }, + }; + } + return undefined; + } + + if (action === "set_chart_view_scale") { + const request = ` ${normalize(this.question)} `; + const styles = actionTool(evidence, action).function.parameters + .properties.styles.items.enum + .filter((/** @type {string} */ style) => + request.includes(` ${normalize(style)} `) + ); + if (styles.length) { + return { + name: action, + arguments: { styles }, + }; + } + } + + if (action !== "explain_evidence") return undefined; + const { context, metricOptions, sourceOptions, guideOptions } = evidence; + const contextual = context.metrics[0] + ? metricOptions.find(({ metric }) => + metric.path === context.metrics[0].path + ) + : undefined; + const mentioned = metricOptions.find(({ origin }) => origin === "mentioned"); + const metric = mentioned ?? contextual; + const grounding = sourceOptions[0] ?? guideOptions[0]; + if (!metric || !grounding) return undefined; + return { + name: action, + arguments: { + refs: [grounding.ref], + metrics: [metric.ref], + }, + }; + } + + /** @param {string} action */ + actionMessages(action) { + const scopedEvidence = this.evidence; + if (!scopedEvidence) throw new Error("Tool session is not ready"); + const { context, apiOptions, sourceOptions, guideOptions } = + scopedEvidence; + const actionMetrics = capabilityMetrics(scopedEvidence, action); + /** @type {Record} */ + const evidence = { + request: this.question, + }; + + if ( + action === "set_chart_view_scale" || + action === "add_chart_series" || + action === "remove_chart_series" || + action === "replace_chart_series" + ) { + evidence.activeChart = context.chart?.chart; + } + if ([ + "read_latest_metric", + "read_metric_at", + "read_metric_range", + "build_metric_chart", + "add_chart_series", + "remove_chart_series", + "replace_chart_series", + "list_metric_variants", + "select_metric_variant", + "explain_evidence", + ].includes(action)) { + evidence.metrics = actionMetrics.map( + (/** @type {any} */ { ref, label, metric, origin }) => ({ + ref, + label, + origin, + path: metric.path, + indexes: action.startsWith("read_metric") || + action === "read_latest_metric" + ? metric.indexes + : undefined, + unit: metric.suggestedUnit, + }), + ); + } + if (action === "explain_evidence" || action === "search_source") { + evidence.source = sourceOptions.map( + (/** @type {any} */ { ref, source }) => ({ + ref, + path: source.path, + startLine: source.startLine, + content: source.content.slice(0, 220), + }), + ); + evidence.guides = guideOptions.map(({ ref, guide }) => ({ + ref, + title: guide.title, + description: guide.description, + })); + if (context.source.length) { + evidence.previousSource = context.source.map( + (/** @type {any} */ source) => ({ + path: source.path, + startLine: source.startLine, + content: source.content, + }), + ); + } + if (context.knowledge) { + evidence.previousAnswer = context.knowledge.description; + } + } + if (action === "call_api") { + evidence.api = apiOptions.map(({ ref, operation }) => ({ + ref, + operation: `${operation.method} ${operation.path}`, + summary: operation.summary || operation.label, + description: operation.description, + parameters: operation.parameters, + response: { + type: operation.response.type, + description: operation.response.description, + fields: operation.response.fields + .slice(0, 12) + .map((/** @type {any} */ { name, type, description }) => ({ + name, + type, + description, + })), + }, + })); + if (context.api) { + evidence.previousApi = { + operation: context.api.operation.label, + arguments: context.api.arguments, + }; + } + } + if ( + (action === "answer_general" || action === "find_chart_metrics") && + context.knowledge + ) { + evidence.context = context.knowledge; + } + return [ + { + role: /** @type {const} */ ("system"), + content: actionInstruction(action), + }, + { + role: /** @type {const} */ ("user"), + content: JSON.stringify(evidence), + }, + ]; + } + + /** @param {string} ref */ + apiArgumentTool(ref) { + if (!this.refs) throw new Error("Tool session is not ready"); + return apiArgumentTool(this.refs.get(ref, "api")); + } + + /** @param {string} ref */ + apiArgumentMessages(ref) { + if (!this.refs) throw new Error("Tool session is not ready"); + const operation = this.refs.get(ref, "api"); + return [ + { + role: /** @type {const} */ ("system"), + content: "Call provide_api_arguments exactly once. Copy only parameter values explicitly supplied by the newest request or verified API context. Omit everything else.", + }, + { + role: /** @type {const} */ ("user"), + content: JSON.stringify({ + request: this.question, + operation: { + method: operation.method, + path: operation.path, + parameters: operation.parameters, + }, + ...(this.evidence?.context.api + ? { + previousArguments: this.evidence.context.api.arguments, + } + : {}), + }), + }, + ]; + } + + /** @param {string} ref */ + apiArguments(ref) { + if (!this.refs) throw new Error("Tool session is not ready"); + const operation = this.refs.get(ref, "api"); + return { + ...(reusableArguments(operation, this.evidence?.context.api) ?? {}), + ...explicitArguments(operation, this.question), + }; + } + + /** @param {string} ref @param {Record} arguments_ */ + hasApiArguments(ref, arguments_) { + if (!this.refs) throw new Error("Tool session is not ready"); + return hasRequiredArguments(this.refs.get(ref, "api"), arguments_); + } + + /** @param {string} ref @param {Record} arguments_ */ + validateApiArguments(ref, arguments_) { + if (!this.refs) throw new Error("Tool session is not ready"); + return validatedArguments( + this.refs.get(ref, "api"), + arguments_, + this.question, + this.evidence?.context.api, + ); + } + + /** @param {string} ref @param {Record} arguments_ */ + missingApiArguments(ref, arguments_) { + if (!this.refs) throw new Error("Tool session is not ready"); + return this.refs.get(ref, "api").parameters + .filter((/** @type {any} */ parameter) => + parameter.required && !Object.hasOwn(arguments_, parameter.name) + ); + } + + /** @param {{ name: string, arguments: Record }} call @param {(status: string) => void} onStatus @param {AbortSignal} signal */ + execute(call, onStatus, signal) { + if (!this.executor) throw new Error("Tool session is not ready"); + return this.executor.execute(call, onStatus, signal); + } +} diff --git a/website_next/ask/tools/source/answer.js b/website_next/ask/tools/source/answer.js deleted file mode 100644 index c78d8a649..000000000 --- a/website_next/ask/tools/source/answer.js +++ /dev/null @@ -1,142 +0,0 @@ -import { normalize } from "../text.js"; - -const ASSIGNMENT = - /\b(?:const|let)\s+([A-Za-z_$][\w$]*)(?:\s*:[^=;]+)?\s*=\s*([^;]{1,320});/gs; - -/** @param {string} value */ -function compact(value) { - return value.replace(/\s+/g, " ").trim(); -} - -/** @param {string} value */ -function words(value) { - return new Set(normalize(value).split(" ").filter(Boolean)); -} - -/** @param {{ content: string }[]} excerpts */ -function assignments(excerpts) { - return excerpts.flatMap((excerpt) => - [...excerpt.content.matchAll(ASSIGNMENT)].map((match) => ({ - name: match[1], - expression: compact(match[2]), - })) - ); -} - -/** @param {{ content: string }[]} excerpts */ -function functions(excerpts) { - return excerpts.flatMap((excerpt) => - [...excerpt.content.matchAll( - /\b(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(|\bfunction\s+([A-Za-z_$][\w$]*)\s*\(/g, - )].map((match) => match[1] ?? match[2]) - ); -} - -/** @param {string} question @param {{ name: string, expression: string }[]} values */ -function primaryAssignment(question, values) { - const queryWords = words(question); - return [...values].sort((left, right) => { - /** @param {{ name: string, expression: string }} value */ - const score = (value) => - Number(queryWords.has(normalize(value.name))) * 10 + - Number(/[+\-*/]/.test(value.expression)) * 3 + - Number(/\b(?:balance|count|price|rate|value)\b/.test(normalize(value.name))); - return score(right) - score(left); - })[0]; -} - -/** - * Produce only answers that can be stated mechanically from the selected - * source. Everything else is left to the model. - * - * @param {string} question - * @param {{ path: string, content: string }[]} excerpts - */ -export function directSourceFact(question, excerpts) { - const text = normalize(question); - const [excerpt] = excerpts; - if (!excerpt) return undefined; - - if ( - /\b(?:file|path)\b/.test(text) || - /^(?:where|which source)\b/.test(text) - ) { - return `The relevant implementation is in \`${excerpt.path}\`.`; - } - - if (/\b(?:function|method)\b/.test(text)) { - const name = functions(excerpts).at(-1); - if (name) return `Start with \`${name}\` in \`${excerpt.path}\`.`; - } - - const values = assignments(excerpts); - if (!values.length) return undefined; - const focus = excerpts.find((item) => - typeof /** @type {{ focus?: unknown }} */ (item).focus === "string" - ); - const primary = primaryAssignment( - `${question} ${/** @type {{ focus?: string }} */ (focus)?.focus ?? ""}`, - values, - ); - if (!primary) return undefined; - - const referencedTerm = ["mempool", "pending"].find((term) => text.includes(term)); - if ( - referencedTerm && - /\b(?:include|includes|including|account|consider|use|uses)\b/.test(text) - ) { - const subject = values.find((value) => - /[+\-*/]/.test(value.expression) && - !normalize(`${value.name} ${value.expression}`).includes(referencedTerm) - ) ?? primary; - if (normalize(subject.expression).includes(referencedTerm)) { - return `Yes. \`${subject.name}\` is computed as \`${subject.expression}\`.`; - } - const separate = values.find((value) => - value !== subject && - /[+\-*/?:]/.test(value.expression) && - normalize(`${value.name} ${value.expression}`).includes(referencedTerm) - ); - if (separate) { - return `No. \`${subject.name}\` is computed as \`${subject.expression}\`. ${referencedTerm === "mempool" ? "Mempool changes are" : "That value is"} handled separately as \`${separate.name} = ${separate.expression}\`.`; - } - return `No. \`${subject.name}\` is computed as \`${subject.expression}\`, which does not reference ${referencedTerm}.`; - } - - if ( - /\b(?:calculat|comput|formula|implement|source)\w*\b/.test(text) || - /^(?:how|what|so)\b/.test(text) - ) { - return `\`${primary.name}\` is computed as \`${primary.expression}\`.`; - } - return undefined; -} - -/** - * @param {string} question - * @param {{ content: string }[]} excerpts - */ -export function sourceFocus(question, excerpts) { - const values = assignments(excerpts); - if (values.length) return primaryAssignment(question, values)?.name; - if (/\b(?:function|method)\b/.test(normalize(question))) { - return functions(excerpts).at(-1); - } - return undefined; -} - -/** - * Exact code expressions are safe to answer from the first source match. - * Location questions still need the resolver to choose among ranked files. - * - * @param {string} question - * @param {{ content: string }[]} excerpts - */ -export function hasDirectSourceComputation(question, excerpts) { - const text = normalize(question); - if ( - /\b(?:file|path)\b/.test(text) || - /^(?:where|which source)\b/.test(text) - ) return false; - return Boolean(directSourceFact(question, /** @type {any} */ (excerpts))); -} diff --git a/website_next/ask/tools/source/formula.js b/website_next/ask/tools/source/formula.js deleted file mode 100644 index 6ee2c2041..000000000 --- a/website_next/ask/tools/source/formula.js +++ /dev/null @@ -1,339 +0,0 @@ -const DEFINITION = /\b([A-Za-z][A-Za-z0-9_]*)\s*=\s*(.+)$/; -const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_]*$/; -const DIFFERENCE = - /^\s*let\s+([A-Za-z][A-Za-z0-9_]*)\s*=\s*\(\s*([A-Za-z][A-Za-z0-9_.]*)\s*-\s*([A-Za-z][A-Za-z0-9_.]*)\s*\)(?:\.as_[A-Za-z0-9_]+\(\)|\s+as\s+[A-Za-z0-9_]+)?\s*;/; -const ACCUMULATION = - /^\s*\*[A-Za-z][A-Za-z0-9_.]*\.([A-Za-z][A-Za-z0-9_]*)\(\)\s*\+=\s*([A-Za-z][A-Za-z0-9_]*)\s*\*\s*([A-Za-z][A-Za-z0-9_.]*)\s*;/; -const CONDITION = - /^(\s*)if\s+([A-Za-z][A-Za-z0-9_.]*)\s*(<=|>=|<|>)\s*([A-Za-z][A-Za-z0-9_.]*)\s*\{/; -const ELSE = /^(\s*)}\s*else\s*\{/; -const INVERSE_OPERATOR = new Map([ - ["<=", ">"], - [">=", "<"], - ["<", ">="], - [">", "<="], -]); -const OPERATOR_TEXT = new Map([ - ["<=", "is at or below"], - [">=", "is at or above"], - ["<", "is below"], - [">", "is above"], -]); - -/** - * @typedef {Object} FormulaFact - * @property {"weighted" | "accumulator" | "ratio"} kind - * @property {string} metric - * @property {string} formula - * @property {string} path - * @property {number} line - * @property {string} [value] - * @property {string} [weight] - * @property {string} [otherWeight] - * @property {string} [summary] - * @property {string} [numerator] - * @property {string} [denominator] - */ - -/** @param {string} value */ -function cleanExpression(value) { - return value - .replace(/[`.;]+$/g, "") - .replace(/[·×]/g, "*") - .replace(/([A-Za-z][A-Za-z0-9_]*)²/g, "$1^2") - .replace(/\s+/g, "") - .replace(/^sum/i, "Σ"); -} - -/** @param {string} value @param {string} operator */ -function splitTopLevel(value, operator) { - let depth = 0; - - for (let index = 0; index < value.length; index += 1) { - const character = value[index]; - if (character === "(") depth += 1; - else if (character === ")") depth -= 1; - else if (character === operator && depth === 0) { - return [value.slice(0, index), value.slice(index + 1)]; - } - } - - return undefined; -} - -/** @param {string} value */ -function unwrapSum(value) { - return value.match(/^Σ\((.+)\)$/)?.[1]; -} - -/** @param {string} value */ -function factorMap(value) { - /** @type {Map} */ - const factors = new Map(); - - for (const rawFactor of value.split("*")) { - const match = rawFactor.match(/^([A-Za-z][A-Za-z0-9_]*)(?:\^(\d+))?$/); - if (!match) return undefined; - - const [, name, rawPower] = match; - const power = Number(rawPower ?? 1); - if (!IDENTIFIER.test(name) || power < 1) return undefined; - factors.set(name, (factors.get(name) ?? 0) + power); - } - - return factors; -} - -/** @param {Map} total @param {Map} part */ -function subtractFactors(total, part) { - const result = new Map(total); - - for (const [name, power] of part) { - const remaining = (result.get(name) ?? 0) - power; - if (remaining < 0) return undefined; - if (remaining === 0) result.delete(name); - else result.set(name, remaining); - } - - return result; -} - -/** @param {Map} factors */ -function factorsText(factors) { - return [...factors] - .flatMap(([name, power]) => Array.from({ length: power }, () => name)) - .join(" × "); -} - -/** @param {string} value */ -function title(value) { - const words = value.replace(/_/g, " "); - if (!words.includes(" ") && words.length <= 4) return words.toUpperCase(); - return words[0].toUpperCase() + words.slice(1); -} - -/** @param {string} value */ -function readableIdentifier(value) { - const identifier = value.split(".").at(-1) ?? value; - return identifier - .replace(/_(?:u|i|f)(?:8|16|32|64|128|size)$/i, "") - .replace(/_/g, " "); -} - -/** @param {string} value */ -function plural(value) { - if (value.includes(" × ")) return `${value} values`; - return value.endsWith("s") ? value : `${value}s`; -} - -/** @param {string} metric @param {string} rawExpression */ -function deriveFormula(metric, rawExpression) { - const expression = cleanExpression(rawExpression); - const division = splitTopLevel(expression, "/"); - if (!division) return undefined; - - const numeratorExpression = unwrapSum(division[0]); - const denominatorExpression = unwrapSum(division[1]); - if (!numeratorExpression || !denominatorExpression) return undefined; - - const numerator = factorMap(numeratorExpression); - const weight = factorMap(denominatorExpression); - if (!numerator || !weight) return undefined; - - const value = subtractFactors(numerator, weight); - if (!value?.size) return undefined; - - const otherWeight = subtractFactors(weight, value); - return { - kind: /** @type {const} */ ("weighted"), - metric, - formula: `${metric} = ${rawExpression.trim()}`, - value: factorsText(value), - weight: factorsText(weight), - otherWeight: otherWeight ? factorsText(otherWeight) : "", - }; -} - -/** @param {{ path: string, text: string }} file */ -function formulasInFile(file) { - const lines = file.text.split("\n"); - /** @type {FormulaFact[]} */ - const comments = lines.flatMap((line, index) => { - const comment = line.replace(/^\s*(?:\/\/\/?|#)\s?/, "").trim(); - const definition = comment.match(DEFINITION); - if (!definition) return []; - - const fact = deriveFormula(definition[1], definition[2]); - return fact ? [{ ...fact, path: file.path, line: index + 1 }] : []; - }); - return [...comments, ...accumulatorFormulas(file.path, lines)]; -} - -/** - * Extract named metrics that are source aliases of the generic price ratio. - * The ratio implementation itself verifies that it is close price / metric price. - * @param {{ path: string, text: string }} file - */ -function ratioAliasesInFile(file) { - const aliases = - file.text.matchAll( - /\blet\s+([A-Za-z][A-Za-z0-9_]*)\s*=\s*LazyPerBlock::from_lazy[\s\S]{0,300}?&([A-Za-z][A-Za-z0-9_]*)\.ratio\b/g, - ); - return [...aliases].map((match) => { - const [, metric, source] = match; - const imported = file.text.match( - new RegExp( - `\\blet\\s+${source}(?:\\s*:[^=;]+)?\\s*=\\s*cfg\\.import\\(\\s*"([^"]+)"`, - ), - )?.[1]; - const denominator = readableIdentifier(imported ?? `${source}_price`); - return { - kind: /** @type {const} */ ("ratio"), - metric, - formula: `${metric} = spot_price / ${imported ?? `${source}_price`}`, - numerator: "spot price", - denominator, - summary: `divides spot price by ${denominator}`, - path: file.path, - line: file.text.slice(0, match.index).split("\n").length, - }; - }); -} - -/** - * @param {string[]} lines - * @param {number} index - */ -function enclosingCondition(lines, index) { - const accumulationIndent = lines[index].match(/^\s*/)?.[0].length ?? 0; - let elseIndent; - for (let cursor = index - 1; cursor >= Math.max(0, index - 16); cursor -= 1) { - const alternate = lines[cursor].match(ELSE); - if (alternate && alternate[1].length < accumulationIndent) { - elseIndent = alternate[1].length; - continue; - } - - const condition = lines[cursor].match(CONDITION); - if (!condition) continue; - const indent = condition[1].length; - if (elseIndent === undefined && indent >= accumulationIndent) continue; - if (elseIndent !== undefined && indent !== elseIndent) continue; - const operator = elseIndent === undefined - ? condition[3] - : INVERSE_OPERATOR.get(condition[3]); - if (!operator) return undefined; - return { - left: readableIdentifier(condition[2]), - operator, - right: readableIdentifier(condition[4]), - }; - } - return undefined; -} - -/** @param {string} path @param {string[]} lines */ -function accumulatorFormulas(path, lines) { - return lines.flatMap((line, index) => { - const accumulation = line.match(ACCUMULATION); - if (!accumulation) return []; - - const [, metric, differenceName, amountName] = accumulation; - let difference; - for (let cursor = index - 1; cursor >= Math.max(0, index - 5); cursor -= 1) { - const candidate = lines[cursor].match(DIFFERENCE); - if (candidate?.[1] === differenceName) { - difference = candidate; - break; - } - } - if (!difference) return []; - - const left = readableIdentifier(difference[2]); - const right = readableIdentifier(difference[3]); - const amount = readableIdentifier(amountName); - const condition = enclosingCondition(lines, index); - const conditionText = condition - ? ` for entries where ${condition.left} ${OPERATOR_TEXT.get(condition.operator)} ${condition.right}` - : ""; - const summary = - `sums \`(${left} − ${right}) × ${amount}\`${conditionText}`; - - return [{ - kind: /** @type {const} */ ("accumulator"), - metric, - formula: `${metric} += (${left} - ${right}) * ${amount}`, - summary, - path, - line: index + 1, - }]; - }); -} - -/** @param {string} question @param {string} metric */ -function questionScore(question, metric) { - const normalizedQuestion = question.toLowerCase().replace(/[_-]+/g, " "); - const normalizedMetric = metric.toLowerCase().replace(/_/g, " "); - if (normalizedQuestion.includes(normalizedMetric)) return normalizedMetric.length + 10; - - const words = normalizedMetric.split(" "); - return words.every((word) => normalizedQuestion.includes(word)) ? words.length : 0; -} - -/** @param {{ path: string, text: string }[]} files */ -export function createFormulaIndex(files) { - const verifiesPriceRatio = files.some(({ text }) => - text.includes("Compute ratio from close price and this metric's price") && - /f64::from\(close\)\s*\/\s*f64::from\(price\)/.test(text) - ); - return /** @type {FormulaFact[]} */ ([ - ...files.flatMap(formulasInFile), - ...(verifiesPriceRatio ? files.flatMap(ratioAliasesInFile) : []), - ]); -} - -/** - * @param {string} question - * @param {ReturnType} formulas - */ -export function explainFormula(question, formulas) { - const fact = formulas - .map((candidate, index) => ({ - candidate, - index, - score: questionScore(question, candidate.metric), - })) - .filter(({ score }) => score > 0) - .sort((a, b) => b.score - a.score || b.index - a.index)[0]?.candidate; - if (!fact) return undefined; - - const metric = title(fact.metric); - if (fact.kind === "accumulator") { - return { - answer: `${metric} ${fact.summary ?? "is accumulated from source values"}.`, - fact, - }; - } - if (fact.kind === "ratio") { - return { - answer: - `${metric} is ${fact.numerator} divided by ${fact.denominator}. ` + - `It compares Bitcoin's current market price with that metric's price basis.`, - fact, - }; - } - - const value = fact.value ?? ""; - const weight = fact.weight ?? ""; - const values = plural(value); - const consequence = fact.otherWeight - ? ` For the same ${fact.otherWeight}, higher ${values} have more influence on the result.` - : ""; - - return { - answer: - `${metric} is a weighted average of ${values}. ` + - `Each ${value} is weighted by \`${weight}\`.${consequence}`, - fact, - }; -} diff --git a/website_next/ask/tools/source/index.js b/website_next/ask/tools/source/index.js index 8a3cefb20..b82da6d8c 100644 --- a/website_next/ask/tools/source/index.js +++ b/website_next/ask/tools/source/index.js @@ -15,37 +15,17 @@ export class AskSource { /** * @param {string} query * @param {string | undefined} path + * @param {"definition" | "implementation" | "availability" | undefined} focus * @param {(progress: { loaded: number, total: number }) => void} onProgress */ - search(query, path, onProgress) { + search(query, path, focus, onProgress) { return this.#client.request( "search", - { query, path }, + { query, path, focus }, ({ loaded, total }) => onProgress({ loaded, total }), ); } - /** - * @param {string} question - * @param {(progress: { loaded: number, total: number }) => void} onProgress - */ - explain(question, onProgress) { - return this.#client.request( - "explain", - { question }, - ({ loaded, total }) => onProgress({ loaded, total }), - ); - } - - /** - * @param {string} path - * @param {number} startLine - * @param {number} endLine - */ - read(path, startLine, endLine) { - return this.#client.request("read", { path, startLine, endLine }); - } - terminate() { this.#client.terminate(); } diff --git a/website_next/ask/tools/source/search.js b/website_next/ask/tools/source/search.js index 35f001e17..d3f3d7d20 100644 --- a/website_next/ask/tools/source/search.js +++ b/website_next/ask/tools/source/search.js @@ -1,29 +1,99 @@ -import { normalize } from "../text.js"; +import { normalize, tokenAffinity } from "../text.js"; -const MAX_EXCERPT_CHARACTERS = 500; +const MAX_EXCERPT_CHARACTERS = 900; const EXCERPT_WINDOW_LINES = 21; -const SOURCE_ALIASES = new Map([ - ["address", ["addr"]], - ["addresses", ["addr", "addrs"]], - ["transaction", ["tx"]], - ["transactions", ["tx", "txs"]], +const TOKEN_AFFINITY = 0.55; +const EXCERPT_CANDIDATES = 64; +const CODE_ALIASES = new Map([ + ["function", ["fn", "func", "def"]], + ["functions", ["fn", "func", "def"]], ]); -/** @param {string} text @param {number} line */ -function excerptAt(text, line) { +/** @param {string[]} words */ +function searchTokens(words) { + return [...new Set(words.flatMap((word) => [ + word, + ...(CODE_ALIASES.get(word) ?? []), + ]))]; +} + +const DECLARATION = /^\s*(?:(?:pub(?:\([^)]*\))?|export|default|async|static)\s+)*(?:fn|function|class|struct|enum|trait|interface|impl)\b/; + +/** @param {string[]} lines */ +function declarations(lines) { + return lines + .map((line, index) => DECLARATION.test(line) ? index : -1) + .filter((index) => index >= 0); +} + +/** @param {string} text @param {number} line @param {number | undefined} declaration */ +function excerptAt(text, line, declaration) { const lines = text.split("\n"); const start = Math.max(1, line - 3); - const end = Math.min(lines.length, line + 3); + const end = Math.min(lines.length, line + 12); + const local = lines.slice(start - 1, end).join("\n"); + const declarationLine = declaration === undefined ? undefined : declaration + 1; + const content = declarationLine !== undefined && declarationLine < start + ? `${lines[declarationLine - 1]}\n...\n${local}` + : local; return { - startLine: start, + startLine: declarationLine !== undefined && declarationLine < start + ? declarationLine + : start, endLine: end, - content: lines - .slice(start - 1, end) - .join("\n") - .slice(0, MAX_EXCERPT_CHARACTERS), + content: content.slice(0, MAX_EXCERPT_CHARACTERS), }; } +/** @param {string} text @param {string} phrase */ +function phraseOccurrences(text, phrase) { + if (!phrase) return 0; + let count = 0; + let index = 0; + while ((index = text.indexOf(phrase, index)) >= 0) { + count += 1; + index += phrase.length; + } + return count; +} + +/** @param {string} content */ +function codeOnly(content) { + return content + .replace(/\/\*[\s\S]*?\*\//g, "") + .split("\n") + .map((line) => { + const trimmed = line.trimStart(); + return trimmed === "*" || + trimmed.startsWith("* ") || + trimmed.startsWith("*/") + ? "" + : line.split("//")[0]; + }) + .join("\n"); +} + +/** @param {string} content */ +function computationWeight(content) { + const code = codeOnly(content); + const arithmetic = code.includes("+=") || + code.includes("-=") || + code.includes("*=") || + code.includes("/=") || + code.includes(" + ") || + code.includes(" - ") || + code.includes(" * ") || + code.includes(" / "); + return arithmetic ? 2 : code.includes(".compute") ? 1 : 0; +} + +/** @param {string} content @param {string} query */ +function computesQueryDirectly(content, query) { + return codeOnly(content).split("\n").some((line) => { + return normalize(line).includes(query) && computationWeight(line) > 0; + }); +} + /** @param {{ path: string, text: string }[]} files */ export function createSourceSearchIndex(files) { /** @type {Map} */ @@ -49,61 +119,72 @@ export function createSourceSearchIndex(files) { /** @param {string} left @param {string} right */ function relatedToken(left, right) { - if (left === right) return true; - /** @param {string} value */ - const stems = (value) => { - const values = new Set([value]); - if (value.length > 5 && value.endsWith("ies")) values.add(`${value.slice(0, -3)}y`); - for (const suffix of ["ation", "tion", "ion", "ing", "ed", "es", "s"]) { - if (value.length - suffix.length >= 4 && value.endsWith(suffix)) { - values.add(value.slice(0, -suffix.length)); - } - } - for (const stem of [...values]) { - if (stem.length >= 5 && stem.endsWith("e")) values.add(stem.slice(0, -1)); - } - return values; - }; - const leftStems = stems(left); - const rightStems = stems(right); - return [...leftStems].some((value) => rightStems.has(value)); + return left === right || tokenAffinity(left, right) >= TOKEN_AFFINITY; } -/** @param {string} text @param {string[]} tokens @param {number[]} weights */ -function bestExcerptLine(text, tokens, weights) { +/** @param {string} text @param {string[]} tokens @param {number[]} weights @param {string} phrase @param {boolean} preferComputation */ +function bestExcerptLine(text, tokens, weights, phrase, preferComputation) { const lines = text.split("\n"); const normalized = lines.map((line) => normalize(line)); + const strongest = Math.max(...weights, 1); + const starts = declarations(lines); + const scopes = starts.length + ? starts.map((start, index) => ({ + start, + end: (starts[index + 1] ?? lines.length) - 1, + })) + : [{ start: 0, end: lines.length - 1 }]; + const rankedScopes = scopes.map((scope) => { + const scopeText = normalized.slice(scope.start, scope.end + 1).join(" "); + const words = new Set( + scopeText.split(" ") + .filter(Boolean), + ); + let score = tokens.reduce((sum, token, tokenIndex) => + [...words].some((word) => relatedToken(word, token)) + ? sum + weights[tokenIndex] + : sum, 0); + if (phrase && scopeText.includes(phrase)) { + score += strongest * 3; + } + if (preferComputation) { + score += computationWeight( + lines.slice(scope.start, scope.end + 1).join("\n"), + ) * strongest * 4; + } + return { ...scope, score }; + }).sort((left, right) => right.score - left.score || left.start - right.start); + const scope = rankedScopes[0]; let bestLine = 1; let bestScore = 0; - for (let start = 0; start < lines.length; start += 1) { - const words = new Set( - normalized - .slice(start, start + EXCERPT_WINDOW_LINES) - .join(" ") - .split(" ") - .filter(Boolean), - ); - const score = tokens.reduce((sum, token, index) => - words.has(token) ? sum + weights[index] : sum, 0); - if (!score || score < bestScore) continue; + for (let index = scope.start; index <= scope.end; index += 1) { + const line = normalized[index]; + const nearby = normalized + .slice( + Math.max(0, index - Math.floor(EXCERPT_WINDOW_LINES / 2)), + index + Math.ceil(EXCERPT_WINDOW_LINES / 2), + ) + .join(" "); + const lineWords = new Set(line.split(" ").filter(Boolean)); + const nearbyWords = new Set(nearby.split(" ").filter(Boolean)); + let score = tokens.reduce((sum, token, tokenIndex) => { + if ([...lineWords].some((word) => relatedToken(word, token))) { + return sum + weights[tokenIndex]; + } + return [...nearbyWords].some((word) => relatedToken(word, token)) + ? sum + weights[tokenIndex] * 0.2 + : sum; + }, 0); + if (phrase && line.includes(phrase)) score += strongest * 3; + if (!score || score <= bestScore) continue; bestScore = score; - bestLine = start + Math.ceil(EXCERPT_WINDOW_LINES / 2); + bestLine = index + 1; } - return Math.min(bestLine, lines.length); -} - -/** @param {string} text @param {string[]} tokens */ -function implementationLine(text, tokens) { - const lines = text.split("\n"); - for (let index = 0; index < lines.length; index += 1) { - const window = lines.slice(index, index + 5).join("\n"); - if (!/[+\-*/]/.test(window)) continue; - if (tokens.some((token) => - new RegExp(`\\b(?:const|let)\\s+${token}\\s*=`).test(lines[index]) - )) return index + 1; - } - return undefined; + const declaration = starts + .filter((start) => start <= bestLine - 1) + .at(-1); + return { line: bestLine, declaration }; } /** @@ -127,18 +208,14 @@ function candidateFiles(index, token) { * @param {ReturnType} index * @param {string} rawQuery * @param {string} [pathPrefix] + * @param {"definition" | "implementation" | "availability"} [focus] */ -export function searchSource(index, rawQuery, pathPrefix = "") { +export function searchSource(index, rawQuery, pathPrefix = "", focus = undefined) { const query = normalize(rawQuery); if (!query) throw new Error("Search query is empty"); const words = query.split(" "); - const tokens = [...new Set( - words.flatMap((word) => [word, ...(SOURCE_ALIASES.get(word) ?? [])]), - )]; - const seeksImplementation = - /\b(?:calculat|comput|formula|implement|source)\w*\b/.test(query) || - /^(?:how|where)\b/.test(query); + const tokens = searchTokens(words); const tokenMatches = tokens .map((token) => ({ token, files: candidateFiles(index, token) })) .filter(({ files }) => files.length) @@ -148,11 +225,6 @@ export function searchSource(index, rawQuery, pathPrefix = "") { weight: Math.log2((index.files.length + 1) / (files.length + 1)) + 1, })); if (!tokenMatches.length) return []; - const maxWeight = Math.max(...tokenMatches.map(({ weight }) => weight)); - const implementationTokens = tokenMatches - .filter(({ weight }) => weight >= maxWeight * 0.45) - .map(({ token }) => token); - /** @type {Map} */ const candidates = new Map(); for (const { token, files, weight } of tokenMatches) { @@ -182,41 +254,22 @@ export function searchSource(index, rawQuery, pathPrefix = "") { pathTokens.some((pathToken) => relatedToken(pathToken, token)) ? score + weight : score, 0); - const implementation = seeksImplementation - ? implementationLine(file.text, implementationTokens) - : undefined; - const line = implementation ?? bestExcerptLine( - file.text, - candidate.queryTokens, - candidate.weights, - ); const semanticScore = [...candidate.weights] .sort((left, right) => right - left) .slice(0, 3) .reduce((sum, weight) => sum + weight, 0); - const excerpt = excerptAt(file.text, line); - const arithmetic = seeksImplementation && - /\blet\s+[a-zA-Z0-9_]+\s*=[^;]{0,300}[+\-*/][^;]*;/s.test(excerpt.content); - const supportFile = /(?:^|\/)(?:tests?|examples?)(?:\/|$)|(?:^|\/)test[_-]/.test(file.path); - const supportPenalty = - seeksImplementation && - supportFile && - !/\b(?:example|test)\w*\b/.test(query) - ? 15 - : 0; matches.push({ + fileIndex, path: file.path, score: semanticScore + - pathScore * 6 + - (exact ? 10 : 0) + - (normalizedPath.includes(query) ? 10 : 0) + - (implementation ? 20 : 0) + - (arithmetic ? 10 : 0) - - supportPenalty, + pathScore * 2 + + (exact ? 30 : 0) + + (normalizedPath.includes(query) ? 15 : 0), matched: candidate.matched, + queryTerms: tokens.length, queryTokens: candidate.queryTokens, + weights: candidate.weights, pathTokens, - ...excerpt, }); } @@ -224,8 +277,55 @@ export function searchSource(index, rawQuery, pathPrefix = "") { right.score - left.score || right.matched - left.matched || left.path.localeCompare(right.path) + ) + .slice(0, EXCERPT_CANDIDATES) + .map((match) => { + const file = index.files[match.fileIndex]; + const { line, declaration } = bestExcerptLine( + file.text, + match.queryTokens, + match.weights, + query, + focus === "implementation", + ); + const excerpt = excerptAt(file.text, line, declaration); + const localPhraseOccurrences = phraseOccurrences( + normalize(excerpt.content), + query, + ); + const declarationText = declaration === undefined + ? "" + : normalize(file.text.split("\n")[declaration]); + const definitionScore = focus === "definition" && + declarationText.includes(query) + ? 60 + : 0; + const implementationScore = focus === "implementation" + ? computationWeight(excerpt.content) * 30 + : 0; + const directImplementationScore = + focus === "implementation" && + computesQueryDirectly(excerpt.content, query) + ? 40 + : 0; + return { + ...match, + score: match.score + + localPhraseOccurrences * 10 + + definitionScore + + implementationScore + + directImplementationScore, + phraseOccurrences: localPhraseOccurrences, + ...excerpt, + }; + }) + .sort((left, right) => + right.score - left.score || + right.phraseOccurrences - left.phraseOccurrences || + right.matched - left.matched || + left.path.localeCompare(right.path) ); - const diversified = ranked.slice(0, 3); + const diversified = ranked.slice(0, 8); const salient = [...tokenMatches] .sort((left, right) => right.weight - left.weight) .slice(0, 5); @@ -242,5 +342,5 @@ export function searchSource(index, rawQuery, pathPrefix = "") { return [...new Map(diversified.map((match) => [match.path, match])).values()] .slice(0, 8) - .map(({ queryTokens, pathTokens, ...match }) => match); + .map(({ fileIndex, queryTokens, weights, pathTokens, ...match }) => match); } diff --git a/website_next/ask/tools/source/worker.js b/website_next/ask/tools/source/worker.js index 5d724112d..306693b82 100644 --- a/website_next/ask/tools/source/worker.js +++ b/website_next/ask/tools/source/worker.js @@ -1,18 +1,10 @@ -import { createFormulaIndex, explainFormula } from "./formula.js"; import { createSourceSearchIndex, searchSource } from "./search.js"; const CATALOG_URL = import.meta.resolve("./catalog.jsonl.gz"); -const MAX_READ_LINES = 120; -const MAX_READ_CHARACTERS = 2_500; -const READ_CONTEXT_BEFORE = 32; -const READ_CONTEXT_AFTER = 40; /** @type {Promise<{ sha: string, files: { path: string, text: string }[] }> | undefined} */ let snapshotPromise; -/** @type {ReturnType | undefined} */ -let formulaIndex; - /** @type {ReturnType | undefined} */ let sourceSearchIndex; @@ -49,8 +41,15 @@ function ensureSnapshot(reportProgress) { return snapshotPromise; } +/** @param {(loaded: number, total: number) => void} reportProgress */ +async function prewarm(reportProgress) { + const snapshot = await ensureSnapshot(reportProgress); + sourceSearchIndex ??= createSourceSearchIndex(snapshot.files); + return true; +} + /** - * @param {{ query: string, path?: string }} args + * @param {{ query: string, path?: string, focus?: "definition" | "implementation" | "availability" }} args * @param {(loaded: number, total: number) => void} reportProgress */ async function search(args, reportProgress) { @@ -59,48 +58,12 @@ async function search(args, reportProgress) { sourceSearchIndex ??= createSourceSearchIndex(snapshot.files); return { revision: snapshot.sha, - matches: searchSource(sourceSearchIndex, String(args.query ?? ""), pathPrefix), - }; -} - -/** - * @param {{ question: string }} args - * @param {(loaded: number, total: number) => void} reportProgress - */ -async function explain(args, reportProgress) { - const snapshot = await ensureSnapshot(reportProgress); - formulaIndex ??= createFormulaIndex(snapshot.files); - const result = explainFormula(String(args.question ?? ""), formulaIndex); - return result ? { revision: snapshot.sha, ...result } : undefined; -} - -/** @param {{ path: string, startLine: number, endLine: number }} args */ -async function read(args) { - const path = String(args.path).replace(/^\/+/, ""); - const selectedStart = Math.max(1, Math.floor(Number(args.startLine))); - const selectedEnd = Math.max(selectedStart, Math.floor(Number(args.endLine))); - const startLine = Math.max(1, selectedStart - READ_CONTEXT_BEFORE); - const endLine = Math.min( - selectedEnd + READ_CONTEXT_AFTER, - startLine + MAX_READ_LINES - 1, - ); - const snapshot = await ensureSnapshot(() => {}); - const text = snapshot.files.find((file) => file.path === path)?.text; - if (text === undefined) throw new Error(`Source file not found: ${path}`); - const lines = text.split("\n"); - if (startLine > lines.length) { - throw new Error(`${path} only has ${lines.length} lines`); - } - const lastLine = Math.min(endLine, lines.length); - const content = lines.slice(startLine - 1, lastLine).join("\n"); - - return { - revision: snapshot.sha, - path, - startLine, - endLine: lastLine, - content: content.slice(0, MAX_READ_CHARACTERS), - truncated: content.length > MAX_READ_CHARACTERS, + matches: searchSource( + sourceSearchIndex, + String(args.query ?? ""), + pathPrefix, + args.focus, + ), }; } @@ -111,13 +74,10 @@ self.addEventListener("message", async (event) => { }; try { - const result = type === "prewarm" - ? await explain({ question: "" }, reportProgress) - : type === "search" - ? await search(data, reportProgress) - : type === "explain" - ? await explain(data, reportProgress) - : await read(data); + let result; + if (type === "prewarm") result = await prewarm(reportProgress); + else if (type === "search") result = await search(data, reportProgress); + else throw new Error(`Unknown source request: ${type}`); self.postMessage({ id, status: "complete", data: result }); } catch (error) { self.postMessage({ id, status: "error", data: errorMessage(error) }); diff --git a/website_next/ask/tools/text.js b/website_next/ask/tools/text.js index c719ae1af..5d5301f7c 100644 --- a/website_next/ask/tools/text.js +++ b/website_next/ask/tools/text.js @@ -53,9 +53,10 @@ export function relevance(query, document, boost = 0) { if (!needle || !haystack) return 0; const words = tokens(needle); - const exact = haystack.includes(needle) ? 30 : 0; + const haystackWords = new Set(tokens(haystack)); + const exact = ` ${haystack} `.includes(` ${needle} `) ? 30 : 0; const coverage = words.length - ? words.filter((word) => haystack.includes(word)).length / words.length + ? words.filter((word) => haystackWords.has(word)).length / words.length : 0; return boost + exact + coverage * 20 + similarity(needle, haystack) * 10; } diff --git a/website_next/ask/worker.js b/website_next/ask/worker.js index 93abf074e..593bb08ba 100644 --- a/website_next/ask/worker.js +++ b/website_next/ask/worker.js @@ -17,6 +17,109 @@ function errorMessage(error) { return error instanceof Error ? error.message : String(error); } +/** @param {string} text */ +function firstJsonObject(text) { + const start = text.indexOf("{"); + if (start < 0) return undefined; + + let depth = 0; + let quoted = false; + let escaped = false; + for (let index = start; index < text.length; index += 1) { + const character = text[index]; + if (quoted) { + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === "\"") quoted = false; + continue; + } + if (character === "\"") quoted = true; + else if (character === "{") depth += 1; + else if (character === "}") { + depth -= 1; + if (depth === 0) return text.slice(start, index + 1); + } + } + return undefined; +} + +/** @param {any} schema @param {unknown} value @returns {unknown} */ +function sanitizeSchemaValue(schema, value) { + if (!schema || typeof schema !== "object") return undefined; + if (schema.type === "string") { + if (typeof value !== "string") return undefined; + return !schema.enum || schema.enum.includes(value) ? value : undefined; + } + if (schema.type === "integer") { + if (typeof value !== "number" || !Number.isInteger(value)) return undefined; + if (schema.minimum !== undefined && value < schema.minimum) return undefined; + if (schema.maximum !== undefined && value > schema.maximum) return undefined; + return value; + } + if (schema.type === "array") { + if (!Array.isArray(value)) return undefined; + const items = /** @type {unknown[]} */ (value + .map((item) => sanitizeSchemaValue(schema.items, item)) + .filter((item) => item !== undefined)); + return [...new Set(items.map((item) => JSON.stringify(item)))] + .map((item) => JSON.parse(item)); + } + if (schema.type === "object") { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const input = /** @type {Record} */ (value); + return Object.fromEntries( + Object.entries(schema.properties ?? {}) + .map(([key, property]) => [ + key, + sanitizeSchemaValue(property, input[key]), + ]) + .filter(([, item]) => item !== undefined), + ); + } + return value; +} + +/** + * BitGPU 0.19.1 can return a valid multi-tool call in text while leaving + * toolCalls empty. Normalize that transport quirk against the supplied schema. + * + * @param {any} result + * @param {readonly any[] | undefined} tools + */ +function normalizedToolCalls(result, tools) { + const available = new Map( + (tools ?? []).map((tool) => [tool.function?.name, tool.function]), + ); + const calls = Array.isArray(result.toolCalls) ? result.toolCalls : []; + const recovered = calls.length + ? calls + : (() => { + const raw = firstJsonObject(String(result.text ?? "")); + if (!raw) return []; + try { + const parsed = JSON.parse(raw); + return typeof parsed.name === "string" && + parsed.arguments && + typeof parsed.arguments === "object" + ? [{ name: parsed.name, arguments: parsed.arguments, raw }] + : []; + } catch { + return []; + } + })(); + + return recovered.flatMap((/** @type {any} */ call) => { + const tool = available.get(call.name); + if (!tool) return []; + const arguments_ = sanitizeSchemaValue( + tool.parameters, + call.arguments, + ); + if (!arguments_ || typeof arguments_ !== "object") return []; + return [{ ...call, arguments: arguments_ }]; + }); +} + /** @param {string} url */ async function cachedResponse(url) { const cache = await caches.open(ASK_MODEL.cacheName); @@ -212,7 +315,7 @@ async function generate(messages, options) { status: "complete", result: { text: result.text, - toolCalls: result.toolCalls, + toolCalls: normalizedToolCalls(result, tools), finishReason: result.finishReason, tokensPerSecond: result.tokensPerSecond, }, diff --git a/website_next/llms-full.txt b/website_next/llms-full.txt index 85ed48b34..a0dc98c56 100644 --- a/website_next/llms-full.txt +++ b/website_next/llms-full.txt @@ -4,7 +4,7 @@ - Version: `v0.3.6` - Base URL: https://bitview.space -- Metrics: 55667 +- Metrics: 56973 - Operations: 97 For machine-readable tool construction, use [https://bitview.space/openapi.json](https://bitview.space/openapi.json). For the complete source-derived series tree, use [https://bitview.space/api/series](https://bitview.space/api/series). @@ -1450,6 +1450,7 @@ curl -s "https://bitview.space/version" ### `AddrChainStats` +- `balance`: `Sats` (required) — Current confirmed balance in satoshis - `funded_txo_count`: `integer` (required) — Total number of transaction outputs that funded this address - `funded_txo_sum`: `Sats` (required) — Total amount in satoshis received by this address across all funded outputs - `spent_txo_count`: `integer` (required) — Total number of transaction outputs spent from this address @@ -1467,6 +1468,7 @@ curl -s "https://bitview.space/version" ### `AddrMempoolStats` +- `balance_delta`: `SatsSigned` (required) — Net unconfirmed balance change in satoshis; negative when pending spends exceed receipts - `funded_txo_count`: `integer` (required) — Number of unconfirmed transaction outputs funding this address - `funded_txo_sum`: `Sats` (required) — Total amount in satoshis being received in unconfirmed transactions - `spent_txo_count`: `integer` (required) — Number of unconfirmed transaction inputs spending from this address @@ -1908,6 +1910,10 @@ curl -s "https://bitview.space/version" `integer` +### `SatsSigned` + +`integer` + ### `SeriesData` - `version`: `Version` (required) — Version of the series data diff --git a/website_next/llms.txt b/website_next/llms.txt index 39c9700fd..8fb278efb 100644 --- a/website_next/llms.txt +++ b/website_next/llms.txt @@ -1,6 +1,6 @@ # Bitcoin Research Kit (BRK) -> Free, open-source Bitcoin analytics API and block explorer. 55667 on-chain time-series and 97 API operations. No authentication required. +> Free, open-source Bitcoin analytics API and block explorer. 56973 on-chain time-series and 97 API operations. No authentication required. ## API