mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-07-23 08:58:11 -07:00
next: ai part 3
This commit is contained in:
nym21
@@ -0,0 +1,66 @@
|
||||
import { colors } from "../../utils/colors.js";
|
||||
import { units } from "../../chart/units.js";
|
||||
import { createMetric } from "./metrics/index.js";
|
||||
|
||||
/** @typedef {import("../storage.js").ChartArtifact} ChartArtifact */
|
||||
|
||||
const VIEWS = new Set(["line", "area", "stacked", "bar", "dots"]);
|
||||
const SCALES = new Set(["linear", "log"]);
|
||||
const PALETTE = ["orange", "blue", "green", "violet", "yellow", "red"];
|
||||
|
||||
/** @param {unknown} value @param {string} name */
|
||||
function requiredString(value, name) {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new Error(`Chart ${name} is required`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
/** @param {Record<string, unknown>} args @returns {ChartArtifact} */
|
||||
export function createChartArtifact(args) {
|
||||
const title = requiredString(args.title, "title");
|
||||
const unit = requiredString(args.unit, "unit");
|
||||
const view = args.view === undefined ? "line" : requiredString(args.view, "view");
|
||||
const scale = args.scale === undefined
|
||||
? "linear"
|
||||
: requiredString(args.scale, "scale");
|
||||
|
||||
if (!Object.hasOwn(units, unit)) throw new Error(`Unknown chart unit: ${unit}`);
|
||||
if (!VIEWS.has(view)) throw new Error(`Unknown chart view: ${view}`);
|
||||
if (!SCALES.has(scale)) throw new Error(`Unknown chart scale: ${scale}`);
|
||||
if (!Array.isArray(args.series) || !args.series.length || args.series.length > 6) {
|
||||
throw new Error("A chart needs between one and six series");
|
||||
}
|
||||
|
||||
const series = args.series.map((value, index) => {
|
||||
if (!value || typeof value !== "object") throw new Error("Invalid chart series");
|
||||
const item = /** @type {Record<string, unknown>} */ (value);
|
||||
const path = requiredString(item.path, "series path")
|
||||
.replace(/^brk\.series\./, "")
|
||||
.replace(/^series\./, "");
|
||||
const label = requiredString(item.label, "series label");
|
||||
const color = item.color === undefined
|
||||
? PALETTE[index]
|
||||
: requiredString(item.color, "series color");
|
||||
|
||||
if (!Object.hasOwn(colors, color)) throw new Error(`Unknown chart color: ${color}`);
|
||||
createMetric(path);
|
||||
return {
|
||||
path,
|
||||
label,
|
||||
color: /** @type {ChartArtifact["chart"]["series"][number]["color"]} */ (color),
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
type: /** @type {const} */ ("chart"),
|
||||
id: crypto.randomUUID(),
|
||||
chart: {
|
||||
title,
|
||||
unit: /** @type {ChartArtifact["chart"]["unit"]} */ (unit),
|
||||
view: /** @type {ChartArtifact["chart"]["view"]} */ (view),
|
||||
scale: /** @type {ChartArtifact["chart"]["scale"]} */ (scale),
|
||||
series,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { brk } from "../../utils/client.js";
|
||||
|
||||
const RANGE_POINTS = 120;
|
||||
|
||||
/** @type {Map<string, Promise<{ indexes: string[], type: string }>>} */
|
||||
const infoCache = new Map();
|
||||
|
||||
/** @param {string} name */
|
||||
function seriesInfo(name) {
|
||||
let request = infoCache.get(name);
|
||||
if (!request) {
|
||||
request = brk.getSeriesInfo(name);
|
||||
infoCache.set(name, request);
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
/** @param {string} type */
|
||||
export function unitFromType(type) {
|
||||
const value = type.toLowerCase();
|
||||
if (value.includes("dollar") || value.includes("usd") || value.includes("cents")) return "usd";
|
||||
if (value.includes("bitcoin") || value === "btc") return "btc";
|
||||
if (value.includes("percent") || value.includes("ratio")) return "percent";
|
||||
if (value.includes("address")) return "addresses";
|
||||
if (value.includes("utxo") || value.includes("output")) return "utxos";
|
||||
if (value.includes("block") || value.includes("height")) return "blocks";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** @param {string} unit */
|
||||
function displayedUnit(unit) {
|
||||
if (unit === "usd") return "$";
|
||||
if (unit === "percent") return "%";
|
||||
if (unit === "btc") return " BTC";
|
||||
if (unit === "addresses") return " addresses";
|
||||
if (unit === "utxos") return " UTXOs";
|
||||
if (unit === "blocks") return " blocks";
|
||||
return "";
|
||||
}
|
||||
|
||||
/** @param {number} value @param {string | undefined} unit */
|
||||
export function formatValue(value, unit) {
|
||||
const number = new Intl.NumberFormat("en-US", { maximumFractionDigits: 8 }).format(value);
|
||||
const affix = displayedUnit(unit ?? "");
|
||||
return affix === "$" ? `$${number}` : `${number}${affix}`;
|
||||
}
|
||||
|
||||
/** @param {{ name: string, suggestedUnit?: string }} metric @param {Record<string, unknown>} action */
|
||||
export async function readMetric(metric, action) {
|
||||
const info = await seriesInfo(metric.name);
|
||||
const rawIndex = typeof action.index === "string" ? action.index : "";
|
||||
const indexLooksLikeValue = /^-?\d+$/.test(rawIndex) || /^\d{4}-\d{2}-\d{2}$/.test(rawIndex);
|
||||
const at = action.at ?? (indexLooksLikeValue ? rawIndex : undefined);
|
||||
const dateLike = typeof at === "string" && !/^-?\d+$/.test(at);
|
||||
const preferredIndex = indexLooksLikeValue ? "" : rawIndex;
|
||||
const index = info.indexes.includes(preferredIndex)
|
||||
? preferredIndex
|
||||
: dateLike && info.indexes.includes("day1")
|
||||
? "day1"
|
||||
: info.indexes.includes("height")
|
||||
? "height"
|
||||
: info.indexes[0];
|
||||
const mode = typeof action.mode === "string" ? action.mode : "latest";
|
||||
let response;
|
||||
|
||||
if (mode === "at") {
|
||||
if (at === undefined) throw new Error("A block height or date is required");
|
||||
response = await brk.getSeries(
|
||||
metric.name,
|
||||
/** @type {any} */ (index),
|
||||
/** @type {any} */ (at),
|
||||
undefined,
|
||||
1,
|
||||
);
|
||||
} else if (mode === "range") {
|
||||
const points = Math.min(Number(action.points) || RANGE_POINTS, RANGE_POINTS);
|
||||
response = await brk.getSeries(
|
||||
metric.name,
|
||||
/** @type {any} */ (index),
|
||||
/** @type {any} */ (action.start ?? -points),
|
||||
/** @type {any} */ (action.end),
|
||||
points,
|
||||
);
|
||||
} else {
|
||||
response = await brk.getSeries(
|
||||
metric.name,
|
||||
/** @type {any} */ (index),
|
||||
-1,
|
||||
undefined,
|
||||
1,
|
||||
);
|
||||
}
|
||||
|
||||
if (!response || typeof response !== "object" || !Array.isArray(response.data)) {
|
||||
throw new Error(`No data returned for ${metric.name}`);
|
||||
}
|
||||
return {
|
||||
name: metric.name,
|
||||
label: metric.name.replaceAll("_", " "),
|
||||
unit: metric.suggestedUnit ?? unitFromType(info.type),
|
||||
index: response.index,
|
||||
start: response.start,
|
||||
end: response.end,
|
||||
stamp: response.stamp,
|
||||
values: response.data,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { searchTool } from "./schemas.js";
|
||||
import { AskToolSession } from "./session.js";
|
||||
import { AskSource } from "./source/index.js";
|
||||
import { terminateMetricIndex } from "./metrics/index.js";
|
||||
|
||||
const MAX_TOOL_ROUNDS = 8;
|
||||
|
||||
/**
|
||||
* @typedef {Object} ToolOutcome
|
||||
* @property {boolean} done
|
||||
* @property {boolean} [general]
|
||||
* @property {string} [output]
|
||||
* @property {import("../storage.js").StoredArtifact[]} [artifacts]
|
||||
*/
|
||||
|
||||
/** @param {import("../model.js").ChatMessage[]} messages */
|
||||
function latestQuestion(messages) {
|
||||
return messages.findLast((message) => message.role === "user")?.content ?? "";
|
||||
}
|
||||
|
||||
export function createAskTools() {
|
||||
const source = new AskSource();
|
||||
/** @type {Map<string, AskToolSession>} */
|
||||
const sessions = new Map();
|
||||
/** @type {AbortController | undefined} */
|
||||
let controller;
|
||||
|
||||
/** @param {string} chatId */
|
||||
function sessionFor(chatId) {
|
||||
let session = sessions.get(chatId);
|
||||
if (!session) {
|
||||
session = new AskToolSession(source);
|
||||
sessions.set(chatId, session);
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
return {
|
||||
toolsFor() {
|
||||
return [searchTool()];
|
||||
},
|
||||
|
||||
/**
|
||||
* @param {Object} options
|
||||
* @param {string} options.chatId
|
||||
* @param {import("../storage.js").StoredMessage[]} options.history
|
||||
* @param {import("../model.js").AskModel} options.model
|
||||
* @param {import("../model.js").ChatMessage[]} options.messages
|
||||
* @param {(update: import("../model.js").TokenUpdate) => void} options.onToken
|
||||
* @param {(status: string) => void} options.onStatus
|
||||
*/
|
||||
async answer({ chatId, history, model, messages, onStatus }) {
|
||||
controller = new AbortController();
|
||||
const { signal } = controller;
|
||||
const question = latestQuestion(messages);
|
||||
const session = sessionFor(chatId);
|
||||
session.begin(question, history);
|
||||
|
||||
try {
|
||||
const direct = await session.tryDirect(onStatus);
|
||||
if (direct) return direct;
|
||||
|
||||
for (let round = 0; round < MAX_TOOL_ROUNDS; round += 1) {
|
||||
signal.throwIfAborted();
|
||||
onStatus("Thinking…");
|
||||
const newestUser = messages.findLast((message) => message.role === "user");
|
||||
const stagedMessages = [
|
||||
{ role: /** @type {const} */ ("system"), content: session.instruction() },
|
||||
...(newestUser ? [newestUser] : []),
|
||||
...(session.observation
|
||||
? [{
|
||||
role: /** @type {const} */ ("user"),
|
||||
content: `Available source-derived result:\n${JSON.stringify(session.observation)}`,
|
||||
}]
|
||||
: []),
|
||||
];
|
||||
const result = await model.generate(
|
||||
stagedMessages,
|
||||
() => {},
|
||||
[await session.tool()],
|
||||
{ name: "next_action" },
|
||||
);
|
||||
const call = result.toolCalls[0];
|
||||
if (!call || call.name !== "next_action") {
|
||||
throw new Error("The AI did not choose a valid action");
|
||||
}
|
||||
|
||||
signal.throwIfAborted();
|
||||
const outcome = /** @type {ToolOutcome} */ (
|
||||
await session.execute(call.arguments, onStatus)
|
||||
);
|
||||
if (!outcome.done) continue;
|
||||
return {
|
||||
output: outcome.output ?? "",
|
||||
artifacts: outcome.artifacts ?? [],
|
||||
};
|
||||
}
|
||||
throw new Error("The AI used too many tool steps. Try a more specific question.");
|
||||
} finally {
|
||||
controller = undefined;
|
||||
onStatus("");
|
||||
}
|
||||
},
|
||||
|
||||
stop() {
|
||||
controller?.abort();
|
||||
source.terminate();
|
||||
},
|
||||
|
||||
terminate() {
|
||||
controller?.abort();
|
||||
controller = undefined;
|
||||
sessions.clear();
|
||||
source.terminate();
|
||||
terminateMetricIndex();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { brk } from "../../utils/client.js";
|
||||
import { relevance } from "./text.js";
|
||||
|
||||
/** @type {Promise<any[]> | undefined} */
|
||||
let catalogPromise;
|
||||
|
||||
/** @param {any} node @param {string[]} breadcrumbs */
|
||||
function record(node, breadcrumbs) {
|
||||
if (!node.chart) return undefined;
|
||||
const series = node.chart.series.flatMap((/** @type {any} */ entry) => {
|
||||
try {
|
||||
const metric = entry.metric(brk);
|
||||
return metric?.name
|
||||
? [{ name: metric.name, label: entry.label }]
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
return {
|
||||
title: node.chart.title ?? node.title,
|
||||
sectionTitle: node.title,
|
||||
breadcrumbs,
|
||||
description: node.description ?? "",
|
||||
unit: node.chart.unit?.id,
|
||||
series,
|
||||
};
|
||||
}
|
||||
|
||||
/** @param {any[]} nodes @param {string[]} ancestors @param {any[]} output */
|
||||
function walk(nodes, ancestors, output) {
|
||||
for (const node of nodes) {
|
||||
const breadcrumbs = [...ancestors, node.title];
|
||||
const item = record(node, breadcrumbs);
|
||||
if (item) output.push(item);
|
||||
if (node.children) walk(node.children, breadcrumbs, output);
|
||||
}
|
||||
}
|
||||
|
||||
async function learnCatalog() {
|
||||
catalogPromise ??= import("../../learn/data/index.js").then(({ sections }) => {
|
||||
/** @type {any[]} */
|
||||
const output = [];
|
||||
walk(sections, [], output);
|
||||
return output;
|
||||
});
|
||||
return catalogPromise;
|
||||
}
|
||||
|
||||
/** @param {string} query @param {number} [limit] */
|
||||
export async function searchLearn(query, limit = 12) {
|
||||
return (await learnCatalog())
|
||||
.map((/** @type {any} */ item) => ({
|
||||
...item,
|
||||
score:
|
||||
relevance(query, `${item.title} ${item.sectionTitle}`) * 3 +
|
||||
relevance(query, item.breadcrumbs.join(" ")) * 2 +
|
||||
relevance(
|
||||
query,
|
||||
`${item.description} ${item.series.flatMap((/** @type {any} */ series) => [series.label, series.name]).join(" ")}`,
|
||||
),
|
||||
}))
|
||||
.filter(({ score }) => score >= 16)
|
||||
.sort((left, right) => right.score - left.score)
|
||||
.slice(0, limit);
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { brk } from "../../../utils/client.js";
|
||||
|
||||
const WORKER_URL = import.meta.resolve("./worker.js");
|
||||
const FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
||||
|
||||
/** @typedef {{ path: string, name: string, endpoint: string, suggestedUnit?: string, matchedQuery?: string, score?: number }} CatalogMetric */
|
||||
|
||||
/** @param {unknown} value */
|
||||
function isMetric(value) {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const by = /** @type {{ by?: Record<string, unknown> }} */ (value).by;
|
||||
return Boolean(
|
||||
by &&
|
||||
Object.values(by).some(
|
||||
(endpoint) =>
|
||||
endpoint &&
|
||||
typeof endpoint === "object" &&
|
||||
typeof /** @type {{ fetch?: unknown }} */ (endpoint).fetch === "function",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
function normalizePath(path) {
|
||||
const normalized = path
|
||||
.trim()
|
||||
.replace(/^brk\.series\./, "")
|
||||
.replace(/^series\./, "");
|
||||
const keys = normalized.split(".").filter(Boolean);
|
||||
|
||||
if (!keys.length || keys.some((key) => FORBIDDEN_KEYS.has(key))) {
|
||||
throw new Error(`Invalid metric path: ${path}`);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
/** @param {typeof brk} client @param {string} path */
|
||||
export function resolveMetric(client, path) {
|
||||
const keys = normalizePath(path);
|
||||
/** @type {unknown} */
|
||||
let value = client.series;
|
||||
|
||||
for (const key of keys) {
|
||||
if (!value || typeof value !== "object" || !Object.hasOwn(value, key)) {
|
||||
throw new Error(`Metric not found: ${path}`);
|
||||
}
|
||||
value = /** @type {Record<string, unknown>} */ (value)[key];
|
||||
}
|
||||
|
||||
if (!isMetric(value)) throw new Error(`Metric not found: ${path}`);
|
||||
return /** @type {TimeframeMetric} */ (value);
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
export function createMetric(path) {
|
||||
resolveMetric(brk, path);
|
||||
return (/** @type {typeof brk} */ client) => resolveMetric(client, path);
|
||||
}
|
||||
|
||||
class MetricIndex {
|
||||
/** @type {Worker | undefined} */
|
||||
#worker;
|
||||
|
||||
/** @type {Map<string, { resolve: (value: any) => void, reject: (error: Error) => void, onProgress?: () => void }>} */
|
||||
#pending = new Map();
|
||||
|
||||
/** @param {"search" | "categories" | "byName" | "variants"} type @param {Record<string, unknown>} data @param {(() => void) | undefined} [onProgress] */
|
||||
request(type, data, onProgress) {
|
||||
this.#ensureWorker();
|
||||
const id = crypto.randomUUID();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.#pending.set(id, { resolve, reject, onProgress });
|
||||
this.#worker?.postMessage({ id, type, data });
|
||||
});
|
||||
}
|
||||
|
||||
#ensureWorker() {
|
||||
if (this.#worker) return;
|
||||
this.#worker = new Worker(WORKER_URL, { type: "module" });
|
||||
this.#worker.addEventListener("message", this.#handleMessage);
|
||||
this.#worker.addEventListener("error", this.#handleError);
|
||||
}
|
||||
|
||||
terminate() {
|
||||
const error = new Error("Metric search stopped");
|
||||
for (const request of this.#pending.values()) request.reject(error);
|
||||
this.#pending.clear();
|
||||
this.#worker?.terminate();
|
||||
this.#worker = undefined;
|
||||
}
|
||||
|
||||
/** @param {MessageEvent} event */
|
||||
#handleMessage = (event) => {
|
||||
const message = event.data;
|
||||
const request = this.#pending.get(message.id);
|
||||
if (!request) return;
|
||||
|
||||
if (message.status === "progress") {
|
||||
request.onProgress?.();
|
||||
return;
|
||||
}
|
||||
|
||||
this.#pending.delete(message.id);
|
||||
if (message.status === "complete") request.resolve(message.data);
|
||||
else request.reject(new Error(message.data));
|
||||
};
|
||||
|
||||
/** @param {ErrorEvent} event */
|
||||
#handleError = (event) => {
|
||||
const error = new Error(event.message || "The metric index failed");
|
||||
for (const request of this.#pending.values()) request.reject(error);
|
||||
this.#pending.clear();
|
||||
this.#worker?.terminate();
|
||||
this.#worker = undefined;
|
||||
};
|
||||
}
|
||||
|
||||
const index = new MetricIndex();
|
||||
|
||||
/** @param {string[]} queries @param {number} [limit] @param {string[]} [prefixes] @param {(() => void) | undefined} [onProgress] @returns {Promise<CatalogMetric[]>} */
|
||||
export function searchMetrics(queries, limit = 16, prefixes = [], onProgress) {
|
||||
return index.request("search", { queries, limit, prefixes }, onProgress);
|
||||
}
|
||||
|
||||
/** @returns {Promise<{ path: string, label: string, count: number, examples: string[] }[]>} */
|
||||
export function metricCategories() {
|
||||
return index.request("categories", {});
|
||||
}
|
||||
|
||||
/** @param {string} name @returns {Promise<CatalogMetric | undefined>} */
|
||||
export function metricByName(name) {
|
||||
return index.request("byName", { name });
|
||||
}
|
||||
|
||||
/** @param {{ name: string }} metric @param {string} query @returns {Promise<{ totalSeries: number, groups: { family: string, examples: string[] }[], series: CatalogMetric[] } | undefined>} */
|
||||
export function metricVariants(metric, query = "") {
|
||||
return index.request("variants", { name: metric.name, query });
|
||||
}
|
||||
|
||||
export function terminateMetricIndex() {
|
||||
index.terminate();
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
import { QuickMatch, QuickMatchConfig } from "../../../modules/quickmatch-js/0.5.0/src/index.js";
|
||||
import { brk } from "../../../utils/client.js";
|
||||
|
||||
const SEARCH_CANDIDATES = 1_024;
|
||||
|
||||
/** @typedef {{ path: string, name: string, endpoint: string, document: string }} CatalogMetric */
|
||||
|
||||
/** @param {unknown} value */
|
||||
function isMetric(value) {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const by = /** @type {{ by?: Record<string, unknown> }} */ (value).by;
|
||||
return Boolean(
|
||||
by &&
|
||||
Object.values(by).some(
|
||||
(endpoint) =>
|
||||
endpoint &&
|
||||
typeof endpoint === "object" &&
|
||||
typeof /** @type {{ fetch?: unknown }} */ (endpoint).fetch === "function",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function searchable(value) {
|
||||
return value
|
||||
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
||||
.replace(/[_./|:-]+/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
function suggestedUnit(path) {
|
||||
if (/(percent|ratio|dominance|rate)/i.test(path)) return "percent";
|
||||
if (/(usd|price|cap)/i.test(path)) return "usd";
|
||||
if (/(btc|supply|value)/i.test(path)) return "btc";
|
||||
if (/(address|addr)/i.test(path)) return "addresses";
|
||||
if (/(utxo|output)/i.test(path)) return "utxos";
|
||||
if (/(block|height|epoch)/i.test(path)) return "blocks";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** @param {number} limit */
|
||||
function createConfig(limit = SEARCH_CANDIDATES) {
|
||||
// The model supplies semantic wording, where unmatched words are usually context rather than typos.
|
||||
return new QuickMatchConfig()
|
||||
.withLimit(limit)
|
||||
.withTrigramBudget(0)
|
||||
.withMinScore(2)
|
||||
.withSeparators("_- :/.|");
|
||||
}
|
||||
|
||||
function buildState() {
|
||||
/** @type {CatalogMetric[]} */
|
||||
const items = [];
|
||||
/** @type {Map<string, CatalogMetric>} */
|
||||
const byName = new Map();
|
||||
/** @type {Map<string, CatalogMetric>} */
|
||||
const byDocument = new Map();
|
||||
const seen = new WeakSet();
|
||||
/** @type {{ value: unknown, keys: string[] }[]} */
|
||||
const pending = [{ value: brk.series, keys: [] }];
|
||||
|
||||
while (pending.length) {
|
||||
const { value, keys } = /** @type {{ value: unknown, keys: string[] }} */ (pending.pop());
|
||||
if (!value || typeof value !== "object" || seen.has(value)) continue;
|
||||
seen.add(value);
|
||||
|
||||
if (isMetric(value)) {
|
||||
const raw = /** @type {{ name?: string, by: Record<string, { path?: string }> }} */ (value);
|
||||
const path = keys.join(".");
|
||||
const name = raw.name ?? keys.at(-1) ?? "";
|
||||
const endpoint = Object.values(raw.by).find((item) => item.path)?.path ?? "";
|
||||
const document = searchable(`${name} | ${path} | ${endpoint}`);
|
||||
const metric = { path, name, endpoint, document };
|
||||
items.push(metric);
|
||||
if (!byName.has(name)) byName.set(name, metric);
|
||||
byDocument.set(document, metric);
|
||||
} else {
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (key !== "by") pending.push({ value: child, keys: [...keys, key] });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const config = createConfig();
|
||||
const matcher = new QuickMatch(items.map(({ document }) => document), config);
|
||||
/** @type {Map<string, { matcher: QuickMatch, config: QuickMatchConfig }>} */
|
||||
const scoped = new Map();
|
||||
return { items, byName, byDocument, matcher, config, scoped };
|
||||
}
|
||||
|
||||
/** @type {Promise<ReturnType<typeof buildState>> | undefined} */
|
||||
let statePromise;
|
||||
|
||||
/** @param {string} id */
|
||||
function state(id) {
|
||||
if (!statePromise) {
|
||||
self.postMessage({ id, status: "progress" });
|
||||
statePromise = Promise.resolve().then(buildState);
|
||||
}
|
||||
return statePromise;
|
||||
}
|
||||
|
||||
/** @param {CatalogMetric} metric */
|
||||
function publicMetric(metric) {
|
||||
return {
|
||||
path: metric.path,
|
||||
name: metric.name,
|
||||
endpoint: metric.endpoint,
|
||||
suggestedUnit: suggestedUnit(metric.path),
|
||||
};
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string[]} prefixes */
|
||||
function scopedIndex(index, prefixes) {
|
||||
const inScope = (/** @type {CatalogMetric} */ metric) =>
|
||||
!prefixes.length || prefixes.some((prefix) =>
|
||||
metric.path === prefix || metric.path.startsWith(`${prefix}.`)
|
||||
);
|
||||
if (!prefixes.length) return { matcher: index.matcher, config: index.config, inScope };
|
||||
|
||||
const key = [...prefixes].sort().join("|");
|
||||
let scoped = index.scoped.get(key);
|
||||
if (!scoped) {
|
||||
const config = createConfig();
|
||||
const documents = index.items.filter(inScope).map(({ document }) => document);
|
||||
scoped = { matcher: new QuickMatch(documents, config), config };
|
||||
index.scoped.set(key, scoped);
|
||||
}
|
||||
return { ...scoped, inScope };
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string} query @param {number} limit @param {string[]} prefixes */
|
||||
function searchOne(index, query, limit, prefixes) {
|
||||
const scope = scopedIndex(index, prefixes);
|
||||
const normalizedQuery = searchable(query);
|
||||
const full = scope.matcher.matchesWith(
|
||||
normalizedQuery,
|
||||
scope.config.withLimit(SEARCH_CANDIDATES),
|
||||
);
|
||||
const fullRanks = new Map(full.map((document, rank) => [document, rank]));
|
||||
const scores = new Map();
|
||||
for (const word of [...new Set(normalizedQuery.split(" "))]) {
|
||||
if (!word) continue;
|
||||
const matches = scope.matcher.matchesWith(
|
||||
word,
|
||||
scope.config.withLimit(SEARCH_CANDIDATES),
|
||||
);
|
||||
for (const [rank, document] of matches.entries()) {
|
||||
const score = scores.get(document) ?? { matched: 0, ranks: 0 };
|
||||
score.matched += 1;
|
||||
score.ranks += rank;
|
||||
scores.set(document, score);
|
||||
}
|
||||
}
|
||||
const documents = [...new Set([...full, ...scores.keys()])]
|
||||
.sort((left, right) => {
|
||||
const a = scores.get(left) ?? { matched: 0, ranks: SEARCH_CANDIDATES };
|
||||
const b = scores.get(right) ?? { matched: 0, ranks: SEARCH_CANDIDATES };
|
||||
return b.matched - a.matched ||
|
||||
(fullRanks.get(left) ?? SEARCH_CANDIDATES) -
|
||||
(fullRanks.get(right) ?? SEARCH_CANDIDATES) ||
|
||||
a.ranks - b.ranks ||
|
||||
left.length - right.length ||
|
||||
left.localeCompare(right);
|
||||
});
|
||||
|
||||
return documents
|
||||
.map((document) => index.byDocument.get(document))
|
||||
.filter((metric) => metric && scope.inScope(metric))
|
||||
.slice(0, limit)
|
||||
.map((metric, rank) => ({
|
||||
...publicMetric(/** @type {CatalogMetric} */ (metric)),
|
||||
matchedQuery: query,
|
||||
score: 1_000 - rank,
|
||||
}));
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string[]} queries @param {number} limit @param {string[]} prefixes */
|
||||
function search(index, queries, limit, prefixes) {
|
||||
const groups = queries.map((query) => searchOne(index, query, limit, prefixes));
|
||||
const output = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (let rank = 0; output.length < limit; rank += 1) {
|
||||
let added = false;
|
||||
for (const group of groups) {
|
||||
const metric = group[rank];
|
||||
if (!metric || seen.has(metric.path)) continue;
|
||||
seen.add(metric.path);
|
||||
output.push(metric);
|
||||
added = true;
|
||||
if (output.length === limit) break;
|
||||
}
|
||||
if (!added) break;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index */
|
||||
function categories(index) {
|
||||
/** @type {Map<string, { path: string, label: string, count: number, branches: Map<string, number>[] }>} */
|
||||
const values = new Map();
|
||||
for (const metric of index.items) {
|
||||
const [root, branch] = metric.path.split(".");
|
||||
if (!root) continue;
|
||||
const category = values.get(root) ?? {
|
||||
path: root,
|
||||
label: searchable(root),
|
||||
count: 0,
|
||||
branches: [new Map(), new Map()],
|
||||
};
|
||||
category.count += 1;
|
||||
if (branch) {
|
||||
category.branches[0].set(branch, (category.branches[0].get(branch) ?? 0) + 1);
|
||||
const selector = metric.path.split(".").slice(1, 3).join(" / ");
|
||||
if (selector !== branch) {
|
||||
category.branches[1].set(selector, (category.branches[1].get(selector) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
values.set(root, category);
|
||||
}
|
||||
return [...values.values()]
|
||||
.map(({ branches, ...category }) => ({
|
||||
...category,
|
||||
examples: branches.flatMap((level, index) => [...level]
|
||||
.sort((left, right) => right[1] - left[1])
|
||||
.slice(0, index === 0 ? 4 : 8)
|
||||
.map(([name]) => searchable(name)))
|
||||
.slice(0, 8),
|
||||
}))
|
||||
.sort((left, right) => left.label.localeCompare(right.label));
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string} name @param {string} query */
|
||||
function variants(index, name, query) {
|
||||
const suffix = `_${name}`;
|
||||
const candidates = index.items.filter((candidate) =>
|
||||
candidate.name === name || candidate.name.endsWith(suffix)
|
||||
);
|
||||
if (candidates.length <= 1) return undefined;
|
||||
|
||||
const preferredPaths = new Map(
|
||||
index.matcher.matchesWith(searchable(query), index.config.withLimit(SEARCH_CANDIDATES))
|
||||
.map((document, rank) => [index.byDocument.get(document)?.path, rank]),
|
||||
);
|
||||
const ranked = candidates
|
||||
.map((candidate) => ({
|
||||
...publicMetric(candidate),
|
||||
rank: preferredPaths.get(candidate.path) ?? SEARCH_CANDIDATES,
|
||||
}))
|
||||
.sort((left, right) =>
|
||||
Number(right.name === name) - Number(left.name === name) ||
|
||||
left.rank - right.rank ||
|
||||
left.path.localeCompare(right.path)
|
||||
);
|
||||
|
||||
let commonSuffix = candidates[0].path.split(".");
|
||||
for (const candidate of candidates.slice(1)) {
|
||||
const path = candidate.path.split(".");
|
||||
let count = 0;
|
||||
while (
|
||||
count < commonSuffix.length &&
|
||||
count < path.length &&
|
||||
commonSuffix[commonSuffix.length - 1 - count] === path[path.length - 1 - count]
|
||||
) count += 1;
|
||||
commonSuffix = count ? commonSuffix.slice(-count) : [];
|
||||
}
|
||||
|
||||
const groups = new Map();
|
||||
for (const candidate of ranked) {
|
||||
const selector = candidate.path.split(".").slice(0, -commonSuffix.length);
|
||||
const cohortIndex = selector.indexOf("cohorts");
|
||||
const cohort = selector.slice(cohortIndex < 0 ? 0 : cohortIndex + 1);
|
||||
const family = cohort.length > 2 ? cohort.slice(0, 2).join(" / ") : cohort[0] ?? "root";
|
||||
const value = cohort.length > 2 ? cohort.slice(2).join(" / ") : cohort[1] ?? "all";
|
||||
const group = groups.get(family) ?? { family, count: 0, examples: [] };
|
||||
group.count += 1;
|
||||
if (group.examples.length < 5) group.examples.push(value);
|
||||
groups.set(family, group);
|
||||
}
|
||||
|
||||
return {
|
||||
totalSeries: ranked.length,
|
||||
groups: [...groups.values()].slice(0, 8),
|
||||
series: ranked.slice(0, 16).map(({ path, name: metricName, endpoint, suggestedUnit }) => ({
|
||||
path,
|
||||
name: metricName,
|
||||
endpoint,
|
||||
suggestedUnit,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
self.addEventListener("message", async (event) => {
|
||||
const { id, type, data } = event.data;
|
||||
try {
|
||||
const index = await state(id);
|
||||
let result;
|
||||
if (type === "search") {
|
||||
result = search(index, data.queries, data.limit, data.prefixes);
|
||||
} else if (type === "categories") {
|
||||
result = categories(index);
|
||||
} else if (type === "byName") {
|
||||
const metric = index.byName.get(data.name);
|
||||
result = metric ? publicMetric(metric) : undefined;
|
||||
} else if (type === "variants") {
|
||||
result = variants(index, data.name, data.query);
|
||||
} else {
|
||||
throw new Error(`Unknown metric request: ${type}`);
|
||||
}
|
||||
self.postMessage({ id, status: "complete", data: result });
|
||||
} catch (error) {
|
||||
self.postMessage({
|
||||
id,
|
||||
status: "error",
|
||||
data: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
const TOOL_CALL_PATTERN = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g;
|
||||
|
||||
/** @param {unknown} value */
|
||||
function normalizeCall(value) {
|
||||
if (!value || typeof value !== "object") {
|
||||
throw new Error("The AI returned an invalid tool call");
|
||||
}
|
||||
|
||||
const call = /** @type {Record<string, unknown>} */ (value);
|
||||
const name = call.name ??
|
||||
(call.function && typeof call.function === "object"
|
||||
? /** @type {Record<string, unknown>} */ (call.function).name
|
||||
: undefined);
|
||||
let args = call.arguments ?? call.args ??
|
||||
(call.function && typeof call.function === "object"
|
||||
? /** @type {Record<string, unknown>} */ (call.function).arguments
|
||||
: undefined);
|
||||
|
||||
if (typeof args === "string") args = JSON.parse(args);
|
||||
if (typeof name !== "string" || !args || typeof args !== "object") {
|
||||
throw new Error("The AI returned an invalid tool call");
|
||||
}
|
||||
|
||||
return { name, arguments: /** @type {Record<string, unknown>} */ (args) };
|
||||
}
|
||||
|
||||
/** @param {string} output */
|
||||
export function parseToolCalls(output) {
|
||||
const matches = [...output.matchAll(TOOL_CALL_PATTERN)];
|
||||
if (matches.length) {
|
||||
return matches.flatMap((match) => {
|
||||
const value = JSON.parse(match[1]);
|
||||
return (Array.isArray(value) ? value : [value]).map(normalizeCall);
|
||||
});
|
||||
}
|
||||
|
||||
if (output.includes("<tool_call")) {
|
||||
throw new Error("The AI returned an incomplete tool call");
|
||||
}
|
||||
|
||||
const candidate = output
|
||||
.trim()
|
||||
.replace(/^```(?:json)?\s*/i, "")
|
||||
.replace(/\s*```$/, "");
|
||||
if (candidate.startsWith("{") && candidate.endsWith("}")) {
|
||||
const value = JSON.parse(candidate);
|
||||
if (value?.name || value?.function?.name) return [normalizeCall(value)];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
const COMMON = `You route one step for Bitview's small on-device Bitcoin assistant.
|
||||
Call next_action exactly once. Never answer directly, invent refs, alter returned refs, or repeat a ref.`;
|
||||
|
||||
export const ASK_STAGE_PROMPTS = /** @type {const} */ ({
|
||||
search: `You route one user request for Bitview's Bitcoin data assistant.
|
||||
Choose the requested outcome and translate the user's meaning into terse catalog-style Bitcoin or BRK metric names or technical noun phrases. Never copy a question or include request verbs, pronouns, time words, or punctuation in a query.
|
||||
Use one query for one metric. For X vs Y, return separate complete X and Y metric phrases; never leave vs or both sides inside one query.
|
||||
Set cardinality to multiple for every comparison or request involving more than one distinct metric, even if you accidentally return one query.
|
||||
Use auto for exact metric terminology. Otherwise choose the narrowest source-derived metric family that expresses the user's meaning. Treat the family as a semantic hint, not part of the query text.
|
||||
Choose reuse_previous when the newest request asks another question about the previous verified topic, including its variants, cohorts, source, value, or chart. Choose extend_previous only when it adds a distinct new metric. Do not turn properties of the previous answer into new metric queries.
|
||||
Choose read_requested_value for a current or historical number.
|
||||
Choose build_requested_chart for a graph, trend, history, comparison over time, or a request to show quantitative metrics over time—even when the user does not say chart.
|
||||
When an active chart is supplied, choose edit_existing_chart only to add, remove, or replace series on that chart.
|
||||
Choose explain_from_verified_facts for what or why questions, meaning, availability, cohorts, variants, or source code.
|
||||
Set focus to variants for cohorts or availability, implementation for source code or calculation details, definition for conceptual explanations, and data for values or charts.
|
||||
Interpret ordinary wording by meaning. BRK means the software repository, not a coin.
|
||||
Call next_action exactly once.`,
|
||||
|
||||
explain: `${COMMON}
|
||||
Choose the smallest sufficient set of returned references by semantic fit and call answer. Prefer recommended references when they answer the request.`,
|
||||
|
||||
navigate: `${COMMON}
|
||||
Choose one to three source-derived metric families that could contain the user's intended meaning. Use their names and examples; normally choose one.
|
||||
Prefer the narrowest ordinary family whose name directly expresses the request. Choose a specialized family only when the user explicitly asks for that specialized concept.`,
|
||||
|
||||
rewrite: `${COMMON}
|
||||
Rewrite the newest request as concise conventional Bitcoin or BRK metric or source-search phrases. Translate colloquial meaning into standard technical terminology. Keep independently requested metrics as separate queries.
|
||||
Return exactly one rewritten query for every supplied unmatched query, in the same order. Never merge comparison sides.
|
||||
When someone asks what one bitcoin is worth or trades for, use "bitcoin spot price". Use cointime only when the user explicitly asks about cointime.
|
||||
Return only those searches through next_action.`,
|
||||
|
||||
read: `${COMMON}
|
||||
Choose the smallest exact metric set that answers the requested value and call read_data.
|
||||
Use latest for the present. Use at for a specific block or date, put that block or date in at, and choose height for a block.`,
|
||||
|
||||
chart: `${COMMON}
|
||||
Build the requested chart from the smallest exact set of returned metric references.
|
||||
Use multiple references only when the user requested a comparison.`,
|
||||
|
||||
editChart: `${COMMON}
|
||||
Edit the active chart with exactly the requested operation and the smallest exact set of returned metric references.`,
|
||||
|
||||
clarify: `${COMMON}
|
||||
Ask one short clarification because the source-derived catalogs did not establish a usable result.`,
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
const PREFIX = /** @type {const} */ ({
|
||||
category: "c",
|
||||
fact: "f",
|
||||
guide: "g",
|
||||
metric: "m",
|
||||
source: "s",
|
||||
});
|
||||
|
||||
export class AskRefs {
|
||||
/** @type {Map<string, number>} */
|
||||
#counts = new Map();
|
||||
|
||||
/** @type {Map<string, { kind: keyof typeof PREFIX, value: any }>} */
|
||||
#items = new Map();
|
||||
|
||||
/** @type {Map<string, string>} */
|
||||
#keys = new Map();
|
||||
|
||||
/**
|
||||
* @param {keyof typeof PREFIX} kind
|
||||
* @param {any} value
|
||||
* @param {string} stableKey
|
||||
*/
|
||||
issue(kind, value, stableKey) {
|
||||
const key = `${kind}:${stableKey}`;
|
||||
const existing = this.#keys.get(key);
|
||||
if (existing) return existing;
|
||||
|
||||
const count = (this.#counts.get(kind) ?? 0) + 1;
|
||||
const ref = `${PREFIX[kind]}${count}`;
|
||||
this.#counts.set(kind, count);
|
||||
this.#items.set(ref, { kind, value });
|
||||
this.#keys.set(key, ref);
|
||||
return ref;
|
||||
}
|
||||
|
||||
/** @param {string} ref @param {keyof typeof PREFIX} [expectedKind] */
|
||||
get(ref, expectedKind) {
|
||||
const item = this.#items.get(ref);
|
||||
if (!item) throw new Error(`Unknown reference ${ref}`);
|
||||
if (expectedKind && item.kind !== expectedKind) {
|
||||
throw new Error(`${ref} is not a ${expectedKind} reference`);
|
||||
}
|
||||
return item.value;
|
||||
}
|
||||
|
||||
/** @param {string} ref */
|
||||
kind(ref) {
|
||||
const item = this.#items.get(ref);
|
||||
if (!item) throw new Error(`Unknown reference ${ref}`);
|
||||
return item.kind;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { formatValue } from "./data.js";
|
||||
|
||||
/**
|
||||
* @typedef {Object} MetricRead
|
||||
* @property {string} label
|
||||
* @property {string | undefined} unit
|
||||
* @property {string} index
|
||||
* @property {number | string} start
|
||||
* @property {string | undefined} stamp
|
||||
* @property {unknown[]} values
|
||||
*/
|
||||
|
||||
/** @param {{ facts: string[], sources: string[], excerpts: { path: string, startLine: number, content: string }[] }} evidence */
|
||||
export function renderEvidence(evidence) {
|
||||
const sections = [...new Set(evidence.facts)].filter(Boolean);
|
||||
for (const excerpt of evidence.excerpts) {
|
||||
sections.push(`\`${excerpt.path}:${excerpt.startLine}\`\n\n\`\`\`\n${excerpt.content}\n\`\`\``);
|
||||
}
|
||||
const sources = [...new Set(evidence.sources)].filter(
|
||||
(source) => !sections.some((section) => section.includes(source)),
|
||||
);
|
||||
if (sources.length) {
|
||||
sections.push(`Source${sources.length === 1 ? "" : "s"}: ${sources.map((source) => `\`${source}\``).join(", ")}`);
|
||||
}
|
||||
return sections.join("\n\n") || "I could not find enough verified evidence to answer that.";
|
||||
}
|
||||
|
||||
/** @param {MetricRead[]} results */
|
||||
export function renderData(results) {
|
||||
return results.map((result) => {
|
||||
if (result.values.length === 1 && typeof result.values[0] === "number") {
|
||||
const position = result.index === "height"
|
||||
? ` at block ${result.start}`
|
||||
: result.stamp
|
||||
? ` at ${result.stamp}`
|
||||
: "";
|
||||
return `**${result.label}**: ${formatValue(result.values[0], result.unit)}${position}`;
|
||||
}
|
||||
const values = /** @type {number[]} */ (
|
||||
result.values.filter((value) => typeof value === "number")
|
||||
);
|
||||
if (!values.length) return `**${result.label}**: no values returned.`;
|
||||
return `**${result.label}**: ${values.length} values; latest ${formatValue(values[values.length - 1], result.unit)}.`;
|
||||
}).join("\n");
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
/** @param {Record<string, unknown>} properties @param {string[]} required */
|
||||
function actionTool(properties, required) {
|
||||
return {
|
||||
type: "function",
|
||||
function: {
|
||||
name: "next_action",
|
||||
description: "Choose exactly one allowed next step.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties,
|
||||
required,
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** @param {boolean} hasActiveChart @param {boolean} hasPrevious @param {{ path: string, label: string }[]} [categories] */
|
||||
export function searchTool(hasActiveChart = false, hasPrevious = false, categories = []) {
|
||||
return actionTool({
|
||||
action: { type: "string", enum: ["search"] },
|
||||
context: {
|
||||
type: "string",
|
||||
enum: hasPrevious
|
||||
? ["new_topic", "reuse_previous", "extend_previous"]
|
||||
: ["new_topic"],
|
||||
description: "Reuse the previous verified topic for dependent follow-ups. Extend it only when the user adds another distinct metric. Otherwise start a new topic.",
|
||||
},
|
||||
queries: {
|
||||
type: "array",
|
||||
minItems: 1,
|
||||
maxItems: 4,
|
||||
items: { type: "string" },
|
||||
description: "One terse catalog-style Bitcoin metric or technical noun phrase per distinct topic. Translate the user's meaning. Never copy a question or include request verbs, pronouns, time words, or punctuation. For X vs Y, return separate complete X and Y metric phrases.",
|
||||
},
|
||||
cardinality: {
|
||||
type: "string",
|
||||
enum: ["single", "multiple"],
|
||||
description: "Use multiple whenever the user requests a comparison or more than one distinct metric, even if queries accidentally contains one item.",
|
||||
},
|
||||
families: {
|
||||
type: "array",
|
||||
minItems: 1,
|
||||
maxItems: 3,
|
||||
items: { type: "string", enum: ["auto", ...categories.map(({ path }) => path)] },
|
||||
description: `Use auto for exact terminology. Otherwise choose the narrowest source-derived family that expresses the user's meaning: ${categories.map(({ path, label }) => `${path}=${label}`).join("; ") || "auto only"}.`,
|
||||
},
|
||||
outcome: {
|
||||
type: "string",
|
||||
enum: [
|
||||
"read_requested_value",
|
||||
"build_requested_chart",
|
||||
...(hasActiveChart ? ["edit_existing_chart"] : []),
|
||||
"explain_from_verified_facts",
|
||||
],
|
||||
},
|
||||
focus: {
|
||||
type: "string",
|
||||
enum: ["definition", "variants", "implementation", "data"],
|
||||
description: "Choose variants for cohorts or availability, implementation for source code or calculation details, definition for conceptual explanations, and data for values or charts.",
|
||||
},
|
||||
}, ["action", "context", "queries", "cardinality", "families", "outcome", "focus"]);
|
||||
}
|
||||
|
||||
/** @param {{ ref: string, label: string }[]} options */
|
||||
export function navigateTool(options) {
|
||||
return actionTool({
|
||||
action: { type: "string", enum: ["navigate"] },
|
||||
refs: {
|
||||
type: "array",
|
||||
minItems: 1,
|
||||
maxItems: 3,
|
||||
items: { type: "string", enum: options.map(({ ref }) => ref) },
|
||||
description: `Metric families: ${options.map(({ ref, label }) => `${ref}=${label}`).join("; ")}`,
|
||||
},
|
||||
}, ["action", "refs"]);
|
||||
}
|
||||
|
||||
/** @param {string[]} queries */
|
||||
export function rewriteTool(queries) {
|
||||
return actionTool({
|
||||
action: { type: "string", enum: ["rewrite"] },
|
||||
queries: {
|
||||
type: "array",
|
||||
minItems: queries.length,
|
||||
maxItems: queries.length,
|
||||
items: { type: "string" },
|
||||
description: `Rewrite each input independently, in the same order, without merging them: ${queries.map((query, index) => `${index + 1}=${query}`).join("; ")}. For what one bitcoin is worth or trades for, use bitcoin spot price. Never use cointime unless explicitly requested.`,
|
||||
},
|
||||
}, ["action", "queries"]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{ ref: string, label: string }[]} options
|
||||
* @param {string} outcome
|
||||
* @param {number} [maxItems]
|
||||
*/
|
||||
export function resolveTool(options, outcome, maxItems = 3) {
|
||||
const refs = {
|
||||
type: "array",
|
||||
minItems: 1,
|
||||
maxItems,
|
||||
items: { type: "string", enum: options.map(({ ref }) => ref) },
|
||||
description: `Available references: ${options.map(({ ref, label }) => `${ref}=${label}`).join("; ")}`,
|
||||
};
|
||||
|
||||
if (outcome === "explain_from_verified_facts") {
|
||||
return actionTool({
|
||||
action: { type: "string", enum: ["answer"] },
|
||||
refs,
|
||||
}, ["action", "refs"]);
|
||||
}
|
||||
|
||||
if (outcome === "read_requested_value") {
|
||||
return actionTool({
|
||||
action: { type: "string", enum: ["read_data"] },
|
||||
refs,
|
||||
mode: { type: "string", enum: ["latest", "at", "range"] },
|
||||
index: { type: "string", description: "Index such as height or day1." },
|
||||
at: { type: "string", description: "Block height or date for at mode." },
|
||||
start: { type: "string" },
|
||||
end: { type: "string" },
|
||||
points: { type: "integer", minimum: 1, maximum: 120 },
|
||||
}, ["action", "refs", "mode"]);
|
||||
}
|
||||
|
||||
const editing = outcome === "edit_existing_chart";
|
||||
return actionTool({
|
||||
action: { type: "string", enum: [editing ? "edit_chart" : "build_chart"] },
|
||||
refs,
|
||||
title: { type: "string" },
|
||||
operation: {
|
||||
type: "string",
|
||||
enum: ["add", "remove", "replace"],
|
||||
description: "For edit_chart, make exactly the requested change.",
|
||||
},
|
||||
}, ["action", "refs"]);
|
||||
}
|
||||
|
||||
export function clarifyTool() {
|
||||
return actionTool({
|
||||
action: { type: "string", enum: ["clarify"] },
|
||||
text: { type: "string", description: "One necessary clarification question." },
|
||||
}, ["action", "text"]);
|
||||
}
|
||||
@@ -0,0 +1,790 @@
|
||||
import { createChartArtifact } from "./chart.js";
|
||||
import { readMetric } from "./data.js";
|
||||
import { searchLearn } from "./learn.js";
|
||||
import {
|
||||
metricByName,
|
||||
metricCategories,
|
||||
metricVariants,
|
||||
searchMetrics,
|
||||
} from "./metrics/index.js";
|
||||
import { ASK_STAGE_PROMPTS } from "./prompts.js";
|
||||
import { AskRefs } from "./refs.js";
|
||||
import { renderData, renderEvidence } from "./render.js";
|
||||
import {
|
||||
clarifyTool,
|
||||
navigateTool,
|
||||
resolveTool,
|
||||
rewriteTool,
|
||||
searchTool,
|
||||
} from "./schemas.js";
|
||||
import { normalize } from "./text.js";
|
||||
|
||||
const MAX_OPTIONS = 12;
|
||||
const MAX_CATEGORIES = 24;
|
||||
|
||||
/** @typedef {import("../storage.js").ChartArtifact} ChartArtifact */
|
||||
|
||||
/**
|
||||
* @typedef {Object} MetricOption
|
||||
* @property {string} ref
|
||||
* @property {string} label
|
||||
* @property {any} metric
|
||||
*/
|
||||
|
||||
/** @param {unknown} value @param {string} name */
|
||||
function requiredString(value, name) {
|
||||
if (typeof value !== "string" || !value.trim()) throw new Error(`${name} is required`);
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
/** @param {unknown} value @param {string} name */
|
||||
function requiredStrings(value, name) {
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
!value.length ||
|
||||
value.some((item) => typeof item !== "string" || !item.trim())
|
||||
) throw new Error(`${name} are required`);
|
||||
return [...new Set(value.map((item) => item.trim()))];
|
||||
}
|
||||
|
||||
/** @param {unknown} value */
|
||||
function uniqueRefs(value) {
|
||||
if (!Array.isArray(value) || !value.length || value.some((ref) => typeof ref !== "string")) {
|
||||
throw new Error("One or more returned references are required");
|
||||
}
|
||||
return [...new Set(value)];
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function label(value) {
|
||||
return value.replaceAll("_", " ");
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function isExplicitComparison(value) {
|
||||
return /\b(?:vs\.?|versus|compare|compared|comparison)\b/i.test(value);
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function referencesPrevious(value) {
|
||||
return /\b(?:it|its|that|this|they|their|them|those|these|same)\b/i.test(value);
|
||||
}
|
||||
|
||||
/** @param {string} request */
|
||||
function isDirectValueFollowup(request) {
|
||||
const text = normalize(request);
|
||||
const hasPoint = /\b(?:current|currently|latest|now|today)\b/.test(text) ||
|
||||
/\bblock\s+\d{4,}\b/.test(text) ||
|
||||
/\b\d{4}-\d{2}-\d{2}\b/.test(text);
|
||||
const needsInterpretation = /^(?:how|why)\b/.test(text) ||
|
||||
/\b(?:available|availability|chart|cohorts?|code|explain|formula|graph|history|plot|source|trend|variants?)\b/.test(text);
|
||||
return referencesPrevious(text) && hasPoint && !needsInterpretation;
|
||||
}
|
||||
|
||||
/** @param {string} request @param {string} proposed */
|
||||
function evidenceFocus(request, proposed) {
|
||||
if (/\b(?:cohorts?|variants?|availability|available)\b/i.test(request)) return "variants";
|
||||
if (/\b(?:source|code|implemented?|implementation|calculated?|calculation|formula)\b/i.test(request)) {
|
||||
return "implementation";
|
||||
}
|
||||
return proposed;
|
||||
}
|
||||
|
||||
/** @param {string} request */
|
||||
function isDirectDefinition(request) {
|
||||
const text = normalize(request);
|
||||
const asks = /\b(?:define|explain|meaning)\b/.test(text) ||
|
||||
/^(?:what is|what are)\b/.test(text);
|
||||
const needsRouting = /\b(?:available|availability|chart|cohorts?|code|current|file|graph|history|latest|now|path|plot|source|today|trend|variants?)\b/.test(text) ||
|
||||
/\bblock\s+\d+\b/.test(text) ||
|
||||
/\b\d{4}-\d{2}-\d{2}\b/.test(text);
|
||||
return asks && !needsRouting;
|
||||
}
|
||||
|
||||
/** @param {string} request */
|
||||
function directReadAction(request) {
|
||||
const block = request.match(/\bblock\s+(\d{4,})\b/i)?.[1];
|
||||
const dates = [...request.matchAll(/\b\d{4}-\d{2}-\d{2}\b/g)].map(([date]) => date);
|
||||
if (block) return { mode: "at", index: "height", at: block };
|
||||
if (dates.length === 1) return { mode: "at", index: "day1", at: dates[0] };
|
||||
if (dates.length > 1 || /\b(?:ago|before|after|between|from|last|previous|since|yesterday)\b/i.test(request)) {
|
||||
return undefined;
|
||||
}
|
||||
return { mode: "latest" };
|
||||
}
|
||||
|
||||
/** @param {string[]} queries */
|
||||
function completeComparisonQueries(queries) {
|
||||
if (queries.length < 3) return queries;
|
||||
|
||||
const shared = queries.at(-1) ?? "";
|
||||
const qualifiers = queries.slice(0, -1);
|
||||
if (!qualifiers.every((query) => normalize(query).split(" ").length === 1)) return queries;
|
||||
|
||||
return qualifiers.map((qualifier) => `${qualifier} ${shared}`);
|
||||
}
|
||||
|
||||
/** @param {any[]} history */
|
||||
function latestChart(history) {
|
||||
for (const message of [...history].reverse()) {
|
||||
const chart = message.artifacts?.findLast?.(
|
||||
(/** @type {any} */ artifact) => artifact.type === "chart",
|
||||
);
|
||||
if (chart) return chart;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** @param {any[]} items */
|
||||
function uniqueMetricOptions(items) {
|
||||
return [...new Map(items.map((/** @type {any} */ item) => [item.ref, item])).values()];
|
||||
}
|
||||
|
||||
/** @param {any[][]} groups */
|
||||
function mergeMetricGroups(groups) {
|
||||
const output = [];
|
||||
const positions = new Map();
|
||||
const ranks = Math.max(...groups.map((group) => group.length), 0);
|
||||
|
||||
for (let rank = 0; rank < ranks && output.length < MAX_OPTIONS; rank += 1) {
|
||||
for (const group of groups) {
|
||||
const metric = group[rank];
|
||||
if (!metric) continue;
|
||||
|
||||
const position = positions.get(metric.path);
|
||||
if (position !== undefined) {
|
||||
const current = output[position];
|
||||
const exact = normalize(metric.name) === normalize(metric.matchedQuery ?? "");
|
||||
const currentExact = normalize(current.name) === normalize(current.matchedQuery ?? "");
|
||||
if (exact && !currentExact) output[position] = metric;
|
||||
continue;
|
||||
}
|
||||
positions.set(metric.path, output.length);
|
||||
output.push(metric);
|
||||
if (output.length === MAX_OPTIONS) break;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
/** @param {any[]} items */
|
||||
function balancedOptions(items) {
|
||||
const order = ["fact", "guide", "metric", "source"];
|
||||
const groups = order
|
||||
.map((kind) => items
|
||||
.filter((item) => item.kind === kind)
|
||||
.sort((left, right) => right.score - left.score))
|
||||
.filter((group) => group.length);
|
||||
const output = [];
|
||||
|
||||
for (let rank = 0; output.length < MAX_OPTIONS; rank += 1) {
|
||||
let added = false;
|
||||
for (const group of groups) {
|
||||
const item = group[rank];
|
||||
if (!item) continue;
|
||||
const { score, ...option } = item;
|
||||
output.push(option);
|
||||
added = true;
|
||||
if (output.length === MAX_OPTIONS) break;
|
||||
}
|
||||
if (!added) break;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export class AskToolSession {
|
||||
/** @param {import("./source/index.js").AskSource} source */
|
||||
constructor(source) {
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
source;
|
||||
refs = new AskRefs();
|
||||
request = "";
|
||||
/** @type {string[]} */
|
||||
previousTopics = [];
|
||||
query = "";
|
||||
/** @type {string[]} */
|
||||
queries = [];
|
||||
outcome = "";
|
||||
focus = "definition";
|
||||
stage = "search";
|
||||
directMatch = false;
|
||||
rewritten = false;
|
||||
rewriteChanged = false;
|
||||
comparison = false;
|
||||
/** @type {string[]} */
|
||||
categoryPaths = [];
|
||||
/** @type {any} */
|
||||
observation;
|
||||
/** @type {any[]} */
|
||||
options = [];
|
||||
/** @type {MetricOption[]} */
|
||||
metricOptions = [];
|
||||
/** @type {any} */
|
||||
formula;
|
||||
sourceSearched = false;
|
||||
/** @type {ChartArtifact | undefined} */
|
||||
activeChart;
|
||||
|
||||
/** @param {string} request @param {any[]} history */
|
||||
begin(request, history) {
|
||||
this.refs = new AskRefs();
|
||||
this.request = request.trim();
|
||||
this.query = "";
|
||||
this.queries = [];
|
||||
this.outcome = "";
|
||||
this.focus = "definition";
|
||||
this.stage = "search";
|
||||
this.directMatch = false;
|
||||
this.rewritten = false;
|
||||
this.rewriteChanged = false;
|
||||
this.comparison = false;
|
||||
this.categoryPaths = [];
|
||||
this.observation = undefined;
|
||||
this.options = [];
|
||||
this.metricOptions = [];
|
||||
this.formula = undefined;
|
||||
this.sourceSearched = false;
|
||||
this.activeChart ??= latestChart(history);
|
||||
}
|
||||
|
||||
/** @param {(status: string) => void} onStatus */
|
||||
async tryDirect(onStatus) {
|
||||
if (this.previousTopics.length === 1 && isDirectValueFollowup(this.request)) {
|
||||
const action = directReadAction(this.request);
|
||||
if (action) {
|
||||
const topic = this.previousTopics[0];
|
||||
const metrics = (await searchMetrics(
|
||||
[topic],
|
||||
3,
|
||||
[],
|
||||
() => onStatus("Indexing metrics…"),
|
||||
)).filter((metric) => normalize(metric.name) === normalize(topic));
|
||||
|
||||
if (metrics.length === 1) {
|
||||
onStatus("Reading data…");
|
||||
const result = await readMetric(metrics[0], action);
|
||||
this.previousTopics = [label(metrics[0].name)];
|
||||
return { output: renderData([result]), artifacts: [] };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!isDirectDefinition(this.request)) return undefined;
|
||||
|
||||
const [metric] = await searchMetrics(
|
||||
[this.request],
|
||||
1,
|
||||
[],
|
||||
() => onStatus("Indexing metrics…"),
|
||||
);
|
||||
if (!metric) return undefined;
|
||||
|
||||
onStatus("Searching source…");
|
||||
const formula = await this.source.explain(
|
||||
`${this.request}\n${metric.name}`,
|
||||
({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`),
|
||||
);
|
||||
if (!formula || normalize(formula.fact.metric) !== normalize(metric.name)) return undefined;
|
||||
|
||||
this.previousTopics = [label(formula.fact.metric)];
|
||||
return { output: formula.answer, artifacts: [] };
|
||||
}
|
||||
|
||||
async tool() {
|
||||
if (this.stage === "search") {
|
||||
const categories = await metricCategories();
|
||||
return searchTool(
|
||||
Boolean(this.activeChart),
|
||||
this.previousTopics.length > 0,
|
||||
categories,
|
||||
);
|
||||
}
|
||||
if (this.stage === "rewrite") return rewriteTool(this.queries);
|
||||
if (this.stage === "navigate") return navigateTool(this.options);
|
||||
if (this.stage === "resolve") {
|
||||
const maxItems = this.formula
|
||||
? 1
|
||||
: this.outcome === "explain_from_verified_facts"
|
||||
? Math.min(this.queries.length, 3)
|
||||
: this.comparison
|
||||
? this.outcome === "read_requested_value" ? 3 : 6
|
||||
: 1;
|
||||
const options = this.outcome === "explain_from_verified_facts"
|
||||
? this.options
|
||||
: this.metricOptions;
|
||||
return resolveTool(options, this.outcome, Math.max(1, maxItems));
|
||||
}
|
||||
return clarifyTool();
|
||||
}
|
||||
|
||||
instruction() {
|
||||
if (this.stage === "search") {
|
||||
const previous = this.previousTopics.length
|
||||
? `\nPrevious verified topic${this.previousTopics.length === 1 ? "" : "s"}: ${this.previousTopics.join(", ")}. Reuse only when the newest request depends on it.`
|
||||
: "";
|
||||
const chart = this.activeChart
|
||||
? `\nActive chart: ${this.activeChart.chart.title}; series: ${this.activeChart.chart.series.map((item) => item.label).join(", ")}.`
|
||||
: "";
|
||||
return `${ASK_STAGE_PROMPTS.search}${previous}${chart}`;
|
||||
}
|
||||
if (this.stage === "navigate") return ASK_STAGE_PROMPTS.navigate;
|
||||
if (this.stage === "rewrite") return ASK_STAGE_PROMPTS.rewrite;
|
||||
if (this.stage === "resolve") {
|
||||
if (this.outcome === "explain_from_verified_facts") {
|
||||
return this.directMatch
|
||||
? `${ASK_STAGE_PROMPTS.explain}\nA trusted direct match exists. Use only recommendedRefs.`
|
||||
: ASK_STAGE_PROMPTS.explain;
|
||||
}
|
||||
if (this.outcome === "read_requested_value") return ASK_STAGE_PROMPTS.read;
|
||||
return this.outcome === "edit_existing_chart"
|
||||
? ASK_STAGE_PROMPTS.editChart
|
||||
: ASK_STAGE_PROMPTS.chart;
|
||||
}
|
||||
return ASK_STAGE_PROMPTS.clarify;
|
||||
}
|
||||
|
||||
/** @param {(status: string) => void} onStatus */
|
||||
async search(onStatus) {
|
||||
const [globalMetrics, rawMetrics, scopedMetrics, guideGroups] = await Promise.all([
|
||||
searchMetrics(
|
||||
this.queries,
|
||||
MAX_OPTIONS,
|
||||
[],
|
||||
() => onStatus("Indexing metrics…"),
|
||||
),
|
||||
searchMetrics(
|
||||
[this.request],
|
||||
MAX_OPTIONS,
|
||||
[],
|
||||
() => onStatus("Indexing metrics…"),
|
||||
),
|
||||
searchMetrics(
|
||||
this.queries,
|
||||
MAX_OPTIONS,
|
||||
this.categoryPaths,
|
||||
() => onStatus("Indexing metrics…"),
|
||||
),
|
||||
Promise.all(this.queries.map((query) => searchLearn(query, MAX_OPTIONS))),
|
||||
]);
|
||||
const foundMetrics = mergeMetricGroups([globalMetrics, rawMetrics, scopedMetrics]);
|
||||
const metrics = [...foundMetrics];
|
||||
const metricNames = new Set(metrics.map((metric) => metric.name));
|
||||
const guides = [...new Map(
|
||||
guideGroups.flat().map((/** @type {any} */ guide) => [guide.breadcrumbs.join("/"), guide]),
|
||||
).values()].filter((/** @type {any} */ guide) =>
|
||||
this.outcome === "explain_from_verified_facts" ||
|
||||
guide.series.some((/** @type {any} */ series) => metricNames.has(series.name))
|
||||
);
|
||||
/** @type {any[]} */
|
||||
let sources = [];
|
||||
|
||||
if (
|
||||
this.outcome === "explain_from_verified_facts" &&
|
||||
this.focus !== "variants" &&
|
||||
!this.sourceSearched
|
||||
) {
|
||||
this.sourceSearched = true;
|
||||
onStatus("Searching source…");
|
||||
const originalMetric = foundMetrics.find((metric) =>
|
||||
normalize(metric.matchedQuery ?? "") === normalize(this.request)
|
||||
);
|
||||
this.formula = await this.source.explain(
|
||||
[this.request, originalMetric?.name, ...this.queries].filter(Boolean).join("\n"),
|
||||
({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`),
|
||||
);
|
||||
if (!this.formula) {
|
||||
sources = (await this.source.search(
|
||||
this.queries.join(" "),
|
||||
undefined,
|
||||
({ loaded, total }) => onStatus(`Indexing source · ${loaded} / ${total}`),
|
||||
)).matches;
|
||||
} else {
|
||||
const formulaMetric = await metricByName(this.formula.fact.metric);
|
||||
if (formulaMetric && !metrics.some((metric) => metric.path === formulaMetric.path)) {
|
||||
metrics.unshift({
|
||||
...formulaMetric,
|
||||
matchedQuery: label(this.formula.fact.metric),
|
||||
score: 2_000,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** @type {any[]} */
|
||||
const ranked = [];
|
||||
for (const metric of metrics) {
|
||||
const ref = this.refs.issue("metric", metric, metric.path);
|
||||
ranked.push({
|
||||
ref,
|
||||
kind: "metric",
|
||||
label: label(metric.name),
|
||||
detail: metric.path,
|
||||
matchedQuery: metric.matchedQuery,
|
||||
score: metric.score,
|
||||
});
|
||||
}
|
||||
for (const guide of guides) {
|
||||
const key = guide.breadcrumbs.join("/");
|
||||
const ref = this.refs.issue("guide", guide, key);
|
||||
ranked.push({
|
||||
ref,
|
||||
kind: "guide",
|
||||
label: guide.title,
|
||||
detail: guide.description,
|
||||
score: guide.score,
|
||||
});
|
||||
}
|
||||
for (const source of sources) {
|
||||
const ref = this.refs.issue("source", source, `${source.path}:${source.startLine}`);
|
||||
ranked.push({
|
||||
ref,
|
||||
kind: "source",
|
||||
label: `${source.path}:${source.startLine}`,
|
||||
detail: source.content,
|
||||
score: source.score,
|
||||
});
|
||||
}
|
||||
if (this.formula && !metrics.length) {
|
||||
const ref = this.refs.issue("fact", this.formula, this.formula.fact.metric);
|
||||
ranked.push({
|
||||
ref,
|
||||
kind: "fact",
|
||||
label: label(this.formula.fact.metric),
|
||||
detail: this.formula.answer,
|
||||
score: 1_000,
|
||||
});
|
||||
}
|
||||
|
||||
this.options = balancedOptions(ranked);
|
||||
if (!this.options.length) {
|
||||
this.stage = "clarify";
|
||||
this.observation = { noMatch: true };
|
||||
return;
|
||||
}
|
||||
|
||||
const normalizedQueries = this.queries.map(normalize);
|
||||
const exactCandidates = this.options.filter((option) =>
|
||||
option.kind === "metric" &&
|
||||
normalize(option.label) === normalize(option.matchedQuery ?? "")
|
||||
);
|
||||
const exact = exactCandidates.filter((option) => {
|
||||
const optionLabel = normalize(option.label);
|
||||
return !exactCandidates.some((candidate) => {
|
||||
const candidateLabel = normalize(candidate.label);
|
||||
return candidateLabel !== optionLabel && candidateLabel.includes(optionLabel);
|
||||
});
|
||||
});
|
||||
const exactByQuery = normalizedQueries.map((query) =>
|
||||
exact.filter((option) => normalize(option.matchedQuery ?? "") === query)
|
||||
);
|
||||
this.directMatch = exactByQuery.every((matches) => matches.length === 1);
|
||||
const directOptions = exactByQuery.flat();
|
||||
const trusted = this.directMatch ||
|
||||
this.comparison && metrics.length > 0 ||
|
||||
this.rewritten && this.rewriteChanged && metrics.length > 0 ||
|
||||
Boolean(this.formula) ||
|
||||
sources.length > 0 ||
|
||||
this.outcome !== "explain_from_verified_facts" && guides.length > 0;
|
||||
if (!trusted && !this.categoryPaths.length) {
|
||||
if (!this.rewritten) {
|
||||
this.stage = "rewrite";
|
||||
this.observation = { unmatchedQueries: this.queries };
|
||||
return;
|
||||
}
|
||||
this.options = (await metricCategories()).slice(0, MAX_CATEGORIES).map((category) => ({
|
||||
ref: this.refs.issue("category", category, category.path),
|
||||
kind: "category",
|
||||
label: category.label,
|
||||
detail: `${category.count} series; ${category.examples.join(", ")}`,
|
||||
}));
|
||||
this.stage = "navigate";
|
||||
this.observation = { options: this.options };
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
this.directMatch &&
|
||||
this.outcome === "explain_from_verified_facts" &&
|
||||
this.focus === "variants"
|
||||
) {
|
||||
onStatus("Inspecting results…");
|
||||
return this.inspect(directOptions.map(({ ref }) => ref));
|
||||
}
|
||||
if (this.directMatch && this.outcome === "read_requested_value") {
|
||||
const action = directReadAction(this.request);
|
||||
if (action) {
|
||||
const refs = directOptions.map(({ ref }) => ref);
|
||||
this.rememberMetrics(refs);
|
||||
onStatus("Reading data…");
|
||||
return this.read({ refs, ...action });
|
||||
}
|
||||
}
|
||||
if (this.directMatch && this.outcome === "build_requested_chart") {
|
||||
const refs = directOptions.map(({ ref }) => ref);
|
||||
this.rememberMetrics(refs);
|
||||
onStatus("Building chart…");
|
||||
return this.buildChart({ refs });
|
||||
}
|
||||
|
||||
this.stage = "resolve";
|
||||
const formulaOption = this.formula
|
||||
? this.options.find((option) =>
|
||||
option.kind === "metric" &&
|
||||
normalize(option.label) === normalize(this.formula.fact.metric)
|
||||
)
|
||||
: undefined;
|
||||
if (this.outcome === "explain_from_verified_facts") {
|
||||
if (formulaOption) return this.inspect([formulaOption.ref]);
|
||||
|
||||
this.observation = {
|
||||
recommendedRefs: (directOptions.length ? directOptions : this.options)
|
||||
.slice(0, 3)
|
||||
.map(({ ref }) => ref),
|
||||
options: this.options,
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
await this.prepareMetricOptions();
|
||||
}
|
||||
|
||||
/** @param {string[]} queries @param {(status: string) => void} onStatus */
|
||||
async rewrite(queries, onStatus) {
|
||||
this.rewriteChanged = queries.length !== this.queries.length ||
|
||||
queries.some((query, index) => normalize(query) !== normalize(this.queries[index] ?? ""));
|
||||
this.queries = queries;
|
||||
this.query = queries.join(" / ");
|
||||
this.rewritten = true;
|
||||
return await this.search(onStatus) ?? { done: false };
|
||||
}
|
||||
|
||||
/** @param {string[]} selected @param {(status: string) => void} onStatus */
|
||||
async navigate(selected, onStatus) {
|
||||
const categories = selected.map((ref) => this.refs.get(ref, "category"));
|
||||
this.categoryPaths = categories.map((category) => category.path);
|
||||
const prefix = categories.map((category) => category.label).join(" ");
|
||||
this.queries = this.queries.map((query) => `${prefix} ${query}`);
|
||||
this.query = this.queries.join(" / ");
|
||||
return await this.search(onStatus) ?? { done: false };
|
||||
}
|
||||
|
||||
async prepareMetricOptions() {
|
||||
/** @type {MetricOption[]} */
|
||||
const metrics = [];
|
||||
|
||||
for (const option of this.options) {
|
||||
const kind = this.refs.kind(option.ref);
|
||||
if (kind === "metric") {
|
||||
const metric = this.refs.get(option.ref, "metric");
|
||||
metrics.push({ ref: option.ref, label: metric.label ?? label(metric.name), metric });
|
||||
} else if (kind === "guide") {
|
||||
const guide = this.refs.get(option.ref, "guide");
|
||||
for (const series of guide.series) {
|
||||
const metric = await metricByName(series.name);
|
||||
if (!metric) continue;
|
||||
const value = { ...metric, label: series.label };
|
||||
const ref = this.refs.issue("metric", value, metric.path);
|
||||
metrics.push({ ref, label: series.label, metric: value });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.metricOptions = uniqueMetricOptions(metrics).slice(0, MAX_OPTIONS);
|
||||
if (!this.metricOptions.length) {
|
||||
this.stage = "clarify";
|
||||
this.observation = { noMetric: true };
|
||||
return;
|
||||
}
|
||||
this.observation = {
|
||||
verifiedMetrics: this.metricOptions.map(({ ref, label: metricLabel, metric }) => ({
|
||||
ref,
|
||||
label: metricLabel,
|
||||
path: metric.path,
|
||||
unit: metric.suggestedUnit,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/** @param {string[]} selected */
|
||||
async inspect(selected) {
|
||||
const evidence = {
|
||||
facts: /** @type {string[]} */ ([]),
|
||||
sources: /** @type {string[]} */ ([]),
|
||||
excerpts: /** @type {any[]} */ ([]),
|
||||
};
|
||||
/** @type {string[]} */
|
||||
const topics = [];
|
||||
|
||||
for (const ref of selected) {
|
||||
const kind = this.refs.kind(ref);
|
||||
if (kind === "fact") {
|
||||
const fact = this.refs.get(ref, "fact");
|
||||
topics.push(label(fact.fact.metric));
|
||||
evidence.facts.push(fact.answer);
|
||||
evidence.sources.push(`${fact.fact.path}:${fact.fact.line}`);
|
||||
} else if (kind === "source") {
|
||||
const match = this.refs.get(ref, "source");
|
||||
const excerpt = await this.source.read(match.path, match.startLine, match.endLine);
|
||||
evidence.excerpts.push(excerpt);
|
||||
evidence.sources.push(`${excerpt.path}:${excerpt.startLine}-${excerpt.endLine}`);
|
||||
} else if (kind === "guide") {
|
||||
const guide = this.refs.get(ref, "guide");
|
||||
if (guide.description) evidence.facts.push(guide.description);
|
||||
topics.push(...guide.series.map((/** @type {any} */ series) => label(series.name)));
|
||||
} else if (kind === "metric") {
|
||||
const metric = this.refs.get(ref, "metric");
|
||||
topics.push(label(metric.name));
|
||||
const variants = this.focus === "variants"
|
||||
? await metricVariants(metric, this.query)
|
||||
: undefined;
|
||||
if (variants) {
|
||||
const groups = variants.groups
|
||||
.map((group) => `${group.family}: ${group.examples.join(", ")}`)
|
||||
.join("; ");
|
||||
evidence.facts.push(
|
||||
`${label(metric.name)} has ${variants.totalSeries} available series. Cohorts: ${groups}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (this.formula && selected.some((ref) => this.refs.kind(ref) === "metric")) {
|
||||
evidence.facts.unshift(this.formula.answer);
|
||||
evidence.sources.push(`${this.formula.fact.path}:${this.formula.fact.line}`);
|
||||
}
|
||||
|
||||
this.previousTopics = [...new Set(topics.length ? topics : this.queries)].slice(0, 4);
|
||||
return { done: true, output: renderEvidence(evidence) };
|
||||
}
|
||||
|
||||
/** @param {Record<string, unknown>} action */
|
||||
async read(action) {
|
||||
const selected = uniqueRefs(action.refs);
|
||||
const block = this.request.match(/\b(?:block\s*)?(\d{4,})\b/i)?.[1];
|
||||
const date = this.request.match(/\b\d{4}-\d{2}-\d{2}\b/)?.[0];
|
||||
const request = action.mode === "at" && action.at === undefined
|
||||
? { ...action, at: block ?? date }
|
||||
: action;
|
||||
const results = await Promise.all(
|
||||
selected.map((ref) => readMetric(this.refs.get(ref, "metric"), request)),
|
||||
);
|
||||
return { done: true, output: renderData(results) };
|
||||
}
|
||||
|
||||
/** @param {Record<string, unknown>} action */
|
||||
buildChart(action) {
|
||||
const selected = uniqueRefs(action.refs);
|
||||
const chosen = selected.map((ref) => this.refs.get(ref, "metric"));
|
||||
const existingChart = this.outcome === "edit_existing_chart"
|
||||
? this.activeChart
|
||||
: undefined;
|
||||
const operation = typeof action.operation === "string" ? action.operation : "add";
|
||||
const added = chosen.map((/** @type {any} */ metric) => ({
|
||||
path: metric.path,
|
||||
label: metric.label ?? label(metric.name),
|
||||
}));
|
||||
const prior = existingChart?.chart.series ?? [];
|
||||
const paths = new Set(added.map(({ path }) => path));
|
||||
const series = !existingChart || operation === "replace"
|
||||
? added
|
||||
: operation === "remove"
|
||||
? prior.filter((item) => !paths.has(item.path))
|
||||
: [...prior, ...added.filter((item) => !prior.some((old) => old.path === item.path))];
|
||||
if (!series.length) throw new Error("A chart needs at least one series");
|
||||
|
||||
const inferredUnit = chosen.map((metric) => metric.suggestedUnit).find(Boolean);
|
||||
const artifact = createChartArtifact({
|
||||
title: typeof action.title === "string" && action.title.trim()
|
||||
? action.title.trim()
|
||||
: existingChart
|
||||
? existingChart.chart.title
|
||||
: chosen.map((metric) => metric.label ?? label(metric.name)).join(" and "),
|
||||
unit: existingChart ? existingChart.chart.unit : inferredUnit,
|
||||
series,
|
||||
});
|
||||
this.activeChart = artifact;
|
||||
return {
|
||||
done: true,
|
||||
output: `${existingChart ? "Updated" : "Built"} **${artifact.chart.title}** with ${artifact.chart.series.map((item) => item.label).join(", ")}.`,
|
||||
artifacts: [artifact],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Record<string, unknown>} action
|
||||
* @param {(status: string) => void} onStatus
|
||||
*/
|
||||
async execute(action, onStatus) {
|
||||
const name = requiredString(action.action, "action");
|
||||
if (this.stage === "search") {
|
||||
if (name !== "search") throw new Error("The AI chose an invalid search action");
|
||||
const context = requiredString(action.context, "context");
|
||||
const proposed = requiredStrings(action.queries, "queries");
|
||||
const cardinality = requiredString(action.cardinality, "cardinality");
|
||||
const families = requiredStrings(action.families, "families")
|
||||
.filter((family) => family !== "auto");
|
||||
const dependent = this.previousTopics.length > 0 && referencesPrevious(this.request);
|
||||
const effectiveContext = dependent
|
||||
? isExplicitComparison(this.request) ? "extend_previous" : "reuse_previous"
|
||||
: context;
|
||||
if (effectiveContext !== "new_topic" && !this.previousTopics.length) {
|
||||
throw new Error("There is no previous verified topic to reuse");
|
||||
}
|
||||
const routedQueries = effectiveContext === "reuse_previous"
|
||||
? [...this.previousTopics]
|
||||
: effectiveContext === "extend_previous"
|
||||
? [...new Set([...this.previousTopics, ...proposed])]
|
||||
: proposed;
|
||||
this.outcome = requiredString(action.outcome, "outcome");
|
||||
this.focus = evidenceFocus(this.request, requiredString(action.focus, "focus"));
|
||||
this.comparison = cardinality === "multiple" || isExplicitComparison(this.request);
|
||||
this.queries = this.comparison
|
||||
? completeComparisonQueries(routedQueries)
|
||||
: routedQueries.slice(0, 1);
|
||||
this.categoryPaths = this.comparison || dependent ? [] : families;
|
||||
this.query = this.queries.join(" / ");
|
||||
return await this.search(onStatus) ?? { done: false };
|
||||
}
|
||||
if (this.stage === "resolve" && this.outcome === "explain_from_verified_facts") {
|
||||
if (name !== "answer") throw new Error("The AI chose an invalid evidence action");
|
||||
onStatus("Inspecting results…");
|
||||
return this.inspect(uniqueRefs(action.refs));
|
||||
}
|
||||
if (this.stage === "navigate") {
|
||||
if (name !== "navigate") throw new Error("The AI chose an invalid navigation action");
|
||||
onStatus("Narrowing search…");
|
||||
return this.navigate(uniqueRefs(action.refs), onStatus);
|
||||
}
|
||||
if (this.stage === "rewrite") {
|
||||
if (name !== "rewrite") throw new Error("The AI chose an invalid rewrite action");
|
||||
onStatus("Refining search…");
|
||||
return this.rewrite(requiredStrings(action.queries, "queries"), onStatus);
|
||||
}
|
||||
if (this.stage === "resolve" && this.outcome === "read_requested_value") {
|
||||
if (name !== "read_data") throw new Error("The AI chose an invalid data action");
|
||||
this.rememberMetrics(uniqueRefs(action.refs));
|
||||
onStatus("Reading data…");
|
||||
return this.read(action);
|
||||
}
|
||||
if (this.stage === "resolve") {
|
||||
if (name !== "build_chart" && name !== "edit_chart") {
|
||||
throw new Error("The AI chose an invalid chart action");
|
||||
}
|
||||
this.rememberMetrics(uniqueRefs(action.refs));
|
||||
onStatus("Building chart…");
|
||||
return this.buildChart(action);
|
||||
}
|
||||
if (name !== "clarify") throw new Error("The AI chose an invalid clarification action");
|
||||
return { done: true, output: requiredString(action.text, "text") };
|
||||
}
|
||||
|
||||
/** @param {string[]} refs */
|
||||
rememberMetrics(refs) {
|
||||
this.previousTopics = refs
|
||||
.map((ref) => label(this.refs.get(ref, "metric").name))
|
||||
.filter((topic, index, topics) => topics.indexOf(topic) === index)
|
||||
.slice(0, 4);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
const DEFINITION = /\b([A-Za-z][A-Za-z0-9_]*)\s*=\s*(.+)$/;
|
||||
const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_]*$/;
|
||||
|
||||
/** @param {string} value */
|
||||
function cleanExpression(value) {
|
||||
return value
|
||||
.replace(/[`.;]+$/g, "")
|
||||
.replace(/[·×]/g, "*")
|
||||
.replace(/([A-Za-z][A-Za-z0-9_]*)²/g, "$1^2")
|
||||
.replace(/\s+/g, "")
|
||||
.replace(/^sum/i, "Σ");
|
||||
}
|
||||
|
||||
/** @param {string} value @param {string} operator */
|
||||
function splitTopLevel(value, operator) {
|
||||
let depth = 0;
|
||||
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const character = value[index];
|
||||
if (character === "(") depth += 1;
|
||||
else if (character === ")") depth -= 1;
|
||||
else if (character === operator && depth === 0) {
|
||||
return [value.slice(0, index), value.slice(index + 1)];
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function unwrapSum(value) {
|
||||
return value.match(/^Σ\((.+)\)$/)?.[1];
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function factorMap(value) {
|
||||
/** @type {Map<string, number>} */
|
||||
const factors = new Map();
|
||||
|
||||
for (const rawFactor of value.split("*")) {
|
||||
const match = rawFactor.match(/^([A-Za-z][A-Za-z0-9_]*)(?:\^(\d+))?$/);
|
||||
if (!match) return undefined;
|
||||
|
||||
const [, name, rawPower] = match;
|
||||
const power = Number(rawPower ?? 1);
|
||||
if (!IDENTIFIER.test(name) || power < 1) return undefined;
|
||||
factors.set(name, (factors.get(name) ?? 0) + power);
|
||||
}
|
||||
|
||||
return factors;
|
||||
}
|
||||
|
||||
/** @param {Map<string, number>} total @param {Map<string, number>} part */
|
||||
function subtractFactors(total, part) {
|
||||
const result = new Map(total);
|
||||
|
||||
for (const [name, power] of part) {
|
||||
const remaining = (result.get(name) ?? 0) - power;
|
||||
if (remaining < 0) return undefined;
|
||||
if (remaining === 0) result.delete(name);
|
||||
else result.set(name, remaining);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** @param {Map<string, number>} factors */
|
||||
function factorsText(factors) {
|
||||
return [...factors]
|
||||
.flatMap(([name, power]) => Array.from({ length: power }, () => name))
|
||||
.join(" × ");
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function title(value) {
|
||||
const words = value.replace(/_/g, " ");
|
||||
return words[0].toUpperCase() + words.slice(1);
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function plural(value) {
|
||||
if (value.includes(" × ")) return `${value} values`;
|
||||
return value.endsWith("s") ? value : `${value}s`;
|
||||
}
|
||||
|
||||
/** @param {string} metric @param {string} rawExpression */
|
||||
function deriveFormula(metric, rawExpression) {
|
||||
const expression = cleanExpression(rawExpression);
|
||||
const division = splitTopLevel(expression, "/");
|
||||
if (!division) return undefined;
|
||||
|
||||
const numeratorExpression = unwrapSum(division[0]);
|
||||
const denominatorExpression = unwrapSum(division[1]);
|
||||
if (!numeratorExpression || !denominatorExpression) return undefined;
|
||||
|
||||
const numerator = factorMap(numeratorExpression);
|
||||
const weight = factorMap(denominatorExpression);
|
||||
if (!numerator || !weight) return undefined;
|
||||
|
||||
const value = subtractFactors(numerator, weight);
|
||||
if (!value?.size) return undefined;
|
||||
|
||||
const otherWeight = subtractFactors(weight, value);
|
||||
return {
|
||||
metric,
|
||||
formula: `${metric} = ${rawExpression.trim()}`,
|
||||
value: factorsText(value),
|
||||
weight: factorsText(weight),
|
||||
otherWeight: otherWeight ? factorsText(otherWeight) : "",
|
||||
};
|
||||
}
|
||||
|
||||
/** @param {{ path: string, text: string }} file */
|
||||
function formulasInFile(file) {
|
||||
return file.text.split("\n").flatMap((line, index) => {
|
||||
const comment = line.replace(/^\s*(?:\/\/\/?|#)\s?/, "").trim();
|
||||
const definition = comment.match(DEFINITION);
|
||||
if (!definition) return [];
|
||||
|
||||
const fact = deriveFormula(definition[1], definition[2]);
|
||||
return fact ? [{ ...fact, path: file.path, line: index + 1 }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {string} question @param {string} metric */
|
||||
function questionScore(question, metric) {
|
||||
const normalizedQuestion = question.toLowerCase().replace(/[_-]+/g, " ");
|
||||
const normalizedMetric = metric.toLowerCase().replace(/_/g, " ");
|
||||
if (normalizedQuestion.includes(normalizedMetric)) return normalizedMetric.length + 10;
|
||||
|
||||
const words = normalizedMetric.split(" ");
|
||||
return words.every((word) => normalizedQuestion.includes(word)) ? words.length : 0;
|
||||
}
|
||||
|
||||
/** @param {{ path: string, text: string }[]} files */
|
||||
export function createFormulaIndex(files) {
|
||||
return files.flatMap(formulasInFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} question
|
||||
* @param {ReturnType<typeof createFormulaIndex>} formulas
|
||||
*/
|
||||
export function explainFormula(question, formulas) {
|
||||
const fact = formulas
|
||||
.map((candidate) => ({
|
||||
candidate,
|
||||
score: questionScore(question, candidate.metric),
|
||||
}))
|
||||
.filter(({ score }) => score > 0)
|
||||
.sort((a, b) => b.score - a.score)[0]?.candidate;
|
||||
if (!fact) return undefined;
|
||||
|
||||
const metric = title(fact.metric);
|
||||
const values = plural(fact.value);
|
||||
const consequence = fact.otherWeight
|
||||
? ` For the same ${fact.otherWeight}, higher ${values} have more influence on the result.`
|
||||
: "";
|
||||
|
||||
return {
|
||||
answer:
|
||||
`${metric} is a weighted average of ${values}. ` +
|
||||
`Each ${fact.value} is weighted by \`${fact.weight}\`.${consequence}\n\n` +
|
||||
`Source: \`${fact.path}:${fact.line}\``,
|
||||
fact,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
const WORKER_URL = import.meta.resolve("./worker.js");
|
||||
|
||||
export class AskSource {
|
||||
/** @type {Worker | undefined} */
|
||||
#worker;
|
||||
|
||||
/** @type {Map<string, { resolve: (value: any) => void, reject: (error: Error) => void, onProgress?: (progress: { loaded: number, total: number }) => void }>} */
|
||||
#pending = new Map();
|
||||
|
||||
/**
|
||||
* @param {string} query
|
||||
* @param {string | undefined} path
|
||||
* @param {(progress: { loaded: number, total: number }) => void} onProgress
|
||||
*/
|
||||
search(query, path, onProgress) {
|
||||
return this.#request("search", { query, path }, onProgress);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} question
|
||||
* @param {(progress: { loaded: number, total: number }) => void} onProgress
|
||||
*/
|
||||
explain(question, onProgress) {
|
||||
return this.#request("explain", { question }, onProgress);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} path
|
||||
* @param {number} startLine
|
||||
* @param {number} endLine
|
||||
*/
|
||||
read(path, startLine, endLine) {
|
||||
return this.#request("read", { path, startLine, endLine });
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {"search" | "read" | "explain"} type
|
||||
* @param {Record<string, unknown>} data
|
||||
* @param {((progress: { loaded: number, total: number }) => void) | undefined} [onProgress]
|
||||
*/
|
||||
#request(type, data, onProgress) {
|
||||
this.#ensureWorker();
|
||||
const id = crypto.randomUUID();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.#pending.set(id, { resolve, reject, onProgress });
|
||||
this.#worker?.postMessage({ id, type, data });
|
||||
});
|
||||
}
|
||||
|
||||
#ensureWorker() {
|
||||
if (this.#worker) return;
|
||||
|
||||
this.#worker = new Worker(WORKER_URL, { type: "module" });
|
||||
this.#worker.addEventListener("message", this.#handleMessage);
|
||||
this.#worker.addEventListener("error", this.#handleWorkerError);
|
||||
}
|
||||
|
||||
terminate() {
|
||||
const error = new Error("Source search stopped");
|
||||
for (const request of this.#pending.values()) request.reject(error);
|
||||
this.#pending.clear();
|
||||
this.#worker?.terminate();
|
||||
this.#worker = undefined;
|
||||
}
|
||||
|
||||
/** @param {MessageEvent} event */
|
||||
#handleMessage = (event) => {
|
||||
const message = event.data;
|
||||
const request = this.#pending.get(message.id);
|
||||
if (!request) return;
|
||||
|
||||
if (message.status === "progress") {
|
||||
request.onProgress?.({ loaded: message.loaded, total: message.total });
|
||||
return;
|
||||
}
|
||||
|
||||
this.#pending.delete(message.id);
|
||||
if (message.status === "complete") request.resolve(message.data);
|
||||
else request.reject(new Error(message.data));
|
||||
};
|
||||
|
||||
/** @param {ErrorEvent} event */
|
||||
#handleWorkerError = (event) => {
|
||||
const error = new Error(event.message || "The source worker failed");
|
||||
for (const request of this.#pending.values()) request.reject(error);
|
||||
this.#pending.clear();
|
||||
this.#worker?.terminate();
|
||||
this.#worker = undefined;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
import { createFormulaIndex, explainFormula } from "./formula.js";
|
||||
|
||||
const REPOSITORY = "bitcoinresearchkit/brk";
|
||||
const TREE_URL = `https://api.github.com/repos/${REPOSITORY}/git/trees/main?recursive=1`;
|
||||
const RAW_URL = `https://raw.githubusercontent.com/${REPOSITORY}`;
|
||||
const DATABASE_NAME = "bitview-ask-source-v1";
|
||||
const STORE_NAME = "snapshots";
|
||||
const MAX_FILE_SIZE = 128_000;
|
||||
const MAX_READ_LINES = 120;
|
||||
const MAX_READ_CHARACTERS = 2_500;
|
||||
const MAX_EXCERPT_CHARACTERS = 500;
|
||||
const CONCURRENCY = 12;
|
||||
const SEARCH_STOPWORDS = new Set([
|
||||
"a",
|
||||
"about",
|
||||
"and",
|
||||
"bitview",
|
||||
"bitcoin",
|
||||
"brk",
|
||||
"code",
|
||||
"define",
|
||||
"does",
|
||||
"explain",
|
||||
"for",
|
||||
"how",
|
||||
"in",
|
||||
"is",
|
||||
"mean",
|
||||
"of",
|
||||
"repo",
|
||||
"repository",
|
||||
"source",
|
||||
"the",
|
||||
"what",
|
||||
"where",
|
||||
"why",
|
||||
"work",
|
||||
"works",
|
||||
]);
|
||||
const EXTENSIONS = new Set([
|
||||
"css",
|
||||
"html",
|
||||
"js",
|
||||
"json",
|
||||
"md",
|
||||
"mjs",
|
||||
"py",
|
||||
"rs",
|
||||
"sh",
|
||||
"toml",
|
||||
"ts",
|
||||
"yaml",
|
||||
"yml",
|
||||
]);
|
||||
const EXCLUDED_PREFIXES = [
|
||||
".git/",
|
||||
".github/",
|
||||
"modules/",
|
||||
"packages/brk_client/brk_client/",
|
||||
"target/",
|
||||
"website/assets/",
|
||||
"website_next/modules/",
|
||||
];
|
||||
const EXCLUDED_FILES = new Set([
|
||||
"docs/CHANGELOG.md",
|
||||
"website/scripts/options/scalar.js",
|
||||
]);
|
||||
|
||||
/** @type {Promise<{ sha: string, entries: { path: string, size: number }[] }> | undefined} */
|
||||
let treePromise;
|
||||
|
||||
/** @type {Promise<{ sha: string, files: { path: string, text: string }[] }> | undefined} */
|
||||
let snapshotPromise;
|
||||
|
||||
/** @type {ReturnType<typeof createFormulaIndex> | undefined} */
|
||||
let formulaIndex;
|
||||
|
||||
/** @param {unknown} error */
|
||||
function errorMessage(error) {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
function extension(path) {
|
||||
return path.slice(path.lastIndexOf(".") + 1).toLowerCase();
|
||||
}
|
||||
|
||||
/** @param {{ type: string, path: string, size?: number }} item */
|
||||
function isSource(item) {
|
||||
return (
|
||||
item.type === "blob" &&
|
||||
Number(item.size ?? 0) <= MAX_FILE_SIZE &&
|
||||
EXTENSIONS.has(extension(item.path)) &&
|
||||
!EXCLUDED_FILES.has(item.path) &&
|
||||
!EXCLUDED_PREFIXES.some((prefix) => item.path.startsWith(prefix))
|
||||
);
|
||||
}
|
||||
|
||||
async function loadTree() {
|
||||
treePromise ??= fetch(TREE_URL, {
|
||||
headers: { Accept: "application/vnd.github+json" },
|
||||
}).then(async (response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`GitHub source tree unavailable (${response.status})`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (data.truncated) throw new Error("GitHub returned a truncated source tree");
|
||||
|
||||
const tree = /** @type {{ type: string, path: string, size?: number }[]} */ (
|
||||
data.tree
|
||||
);
|
||||
return {
|
||||
sha: data.sha,
|
||||
entries: tree.filter(isSource).map((item) => ({
|
||||
path: item.path,
|
||||
size: Number(item.size ?? 0),
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
return treePromise;
|
||||
}
|
||||
|
||||
/** @param {string} sha @param {string} path */
|
||||
async function fetchSource(sha, path) {
|
||||
const encodedPath = path.split("/").map(encodeURIComponent).join("/");
|
||||
const response = await fetch(`${RAW_URL}/${sha}/${encodedPath}`);
|
||||
if (!response.ok) throw new Error(`Could not fetch ${path}`);
|
||||
return response.text();
|
||||
}
|
||||
|
||||
function openDatabase() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = indexedDB.open(DATABASE_NAME, 1);
|
||||
request.addEventListener("upgradeneeded", () => {
|
||||
request.result.createObjectStore(STORE_NAME, { keyPath: "sha" });
|
||||
});
|
||||
request.addEventListener("success", () => resolve(request.result));
|
||||
request.addEventListener("error", () => reject(request.error));
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {IDBDatabase} database @param {string} sha */
|
||||
function readSnapshot(database, sha) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = database
|
||||
.transaction(STORE_NAME, "readonly")
|
||||
.objectStore(STORE_NAME)
|
||||
.get(sha);
|
||||
request.addEventListener("success", () => resolve(request.result));
|
||||
request.addEventListener("error", () => reject(request.error));
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {IDBDatabase} database @param {{ sha: string, files: { path: string, text: string }[] }} snapshot */
|
||||
function writeSnapshot(database, snapshot) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const transaction = database.transaction(STORE_NAME, "readwrite");
|
||||
const store = transaction.objectStore(STORE_NAME);
|
||||
store.clear();
|
||||
store.put(snapshot);
|
||||
transaction.addEventListener("complete", () => resolve(undefined));
|
||||
transaction.addEventListener("error", () => reject(transaction.error));
|
||||
});
|
||||
}
|
||||
|
||||
/** @param {(loaded: number, total: number) => void} reportProgress */
|
||||
async function loadSnapshot(reportProgress) {
|
||||
const tree = await loadTree();
|
||||
const database = /** @type {IDBDatabase} */ (await openDatabase());
|
||||
|
||||
try {
|
||||
const cached = /** @type {{ sha: string, files: { path: string, text: string }[] } | undefined} */ (
|
||||
await readSnapshot(database, tree.sha)
|
||||
);
|
||||
if (cached) {
|
||||
reportProgress(cached.files.length, cached.files.length);
|
||||
return cached;
|
||||
}
|
||||
|
||||
const files = new Array(tree.entries.length);
|
||||
let cursor = 0;
|
||||
let loaded = 0;
|
||||
reportProgress(loaded, tree.entries.length);
|
||||
|
||||
async function download() {
|
||||
while (cursor < tree.entries.length) {
|
||||
const index = cursor;
|
||||
cursor += 1;
|
||||
const entry = tree.entries[index];
|
||||
files[index] = {
|
||||
path: entry.path,
|
||||
text: await fetchSource(tree.sha, entry.path),
|
||||
};
|
||||
loaded += 1;
|
||||
if (loaded % 20 === 0 || loaded === tree.entries.length) {
|
||||
reportProgress(loaded, tree.entries.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(CONCURRENCY, tree.entries.length) }, download),
|
||||
);
|
||||
const snapshot = { sha: tree.sha, files };
|
||||
await writeSnapshot(database, snapshot);
|
||||
return snapshot;
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {(loaded: number, total: number) => void} reportProgress */
|
||||
function ensureSnapshot(reportProgress) {
|
||||
snapshotPromise ??= loadSnapshot(reportProgress).catch((error) => {
|
||||
snapshotPromise = undefined;
|
||||
throw error;
|
||||
});
|
||||
return snapshotPromise;
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function normalize(value) {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** @param {string} text @param {number} index */
|
||||
function lineAt(text, index) {
|
||||
return text.slice(0, index).split("\n").length;
|
||||
}
|
||||
|
||||
/** @param {string} text @param {number} line */
|
||||
function excerptAt(text, line) {
|
||||
const lines = text.split("\n");
|
||||
const start = Math.max(1, line - 3);
|
||||
const end = Math.min(lines.length, line + 3);
|
||||
return {
|
||||
startLine: start,
|
||||
endLine: end,
|
||||
content: lines
|
||||
.slice(start - 1, end)
|
||||
.join("\n")
|
||||
.slice(0, MAX_EXCERPT_CHARACTERS),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{ query: string, path?: string }} args
|
||||
* @param {(loaded: number, total: number) => void} reportProgress
|
||||
*/
|
||||
async function search(args, reportProgress) {
|
||||
const rawQuery = normalize(args.query);
|
||||
if (!rawQuery) throw new Error("Search query is empty");
|
||||
const pathPrefix = String(args.path ?? "").replace(/^\/+/, "");
|
||||
const rawTokens = rawQuery.split(" ");
|
||||
const tokens = rawTokens.filter((token) => !SEARCH_STOPWORDS.has(token));
|
||||
const searchTokens = tokens.length ? tokens : rawTokens;
|
||||
const query = searchTokens.join(" ");
|
||||
const snapshot = await ensureSnapshot(reportProgress);
|
||||
const matches = [];
|
||||
|
||||
for (const file of snapshot.files) {
|
||||
if (pathPrefix && !file.path.startsWith(pathPrefix)) continue;
|
||||
const normalized = normalize(file.text);
|
||||
let index = normalized.indexOf(query);
|
||||
let score = 2;
|
||||
|
||||
if (index < 0 && searchTokens.every((token) => normalized.includes(token))) {
|
||||
index = normalized.indexOf(searchTokens[0]);
|
||||
score = 1;
|
||||
}
|
||||
if (index < 0) continue;
|
||||
|
||||
const rawIndex = file.text.toLowerCase().indexOf(searchTokens[0]);
|
||||
const line = lineAt(file.text, Math.max(0, rawIndex));
|
||||
matches.push({
|
||||
path: file.path,
|
||||
score: score + (normalize(file.path).includes(query) ? 2 : 0),
|
||||
...excerptAt(file.text, line),
|
||||
});
|
||||
}
|
||||
|
||||
matches.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
|
||||
return {
|
||||
revision: snapshot.sha,
|
||||
matches: matches.slice(0, 4).map(({ score, ...match }) => match),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{ question: string }} args
|
||||
* @param {(loaded: number, total: number) => void} reportProgress
|
||||
*/
|
||||
async function explain(args, reportProgress) {
|
||||
const snapshot = await ensureSnapshot(reportProgress);
|
||||
formulaIndex ??= createFormulaIndex(snapshot.files);
|
||||
const result = explainFormula(String(args.question ?? ""), formulaIndex);
|
||||
return result ? { revision: snapshot.sha, ...result } : undefined;
|
||||
}
|
||||
|
||||
/** @param {{ path: string, startLine: number, endLine: number }} args */
|
||||
async function read(args) {
|
||||
const path = String(args.path).replace(/^\/+/, "");
|
||||
const startLine = Math.max(1, Math.floor(Number(args.startLine)));
|
||||
const requestedEnd = Math.max(startLine, Math.floor(Number(args.endLine)));
|
||||
const endLine = Math.min(requestedEnd, startLine + MAX_READ_LINES - 1);
|
||||
const tree = await loadTree();
|
||||
if (!tree.entries.some((entry) => entry.path === path)) {
|
||||
throw new Error(`Source file not found: ${path}`);
|
||||
}
|
||||
|
||||
const snapshot = await snapshotPromise;
|
||||
const text = snapshot?.files.find((file) => file.path === path)?.text ??
|
||||
(await fetchSource(tree.sha, path));
|
||||
const lines = text.split("\n");
|
||||
if (startLine > lines.length) {
|
||||
throw new Error(`${path} only has ${lines.length} lines`);
|
||||
}
|
||||
const lastLine = Math.min(endLine, lines.length);
|
||||
const content = lines.slice(startLine - 1, lastLine).join("\n");
|
||||
|
||||
return {
|
||||
revision: tree.sha,
|
||||
path,
|
||||
startLine,
|
||||
endLine: lastLine,
|
||||
content: content.slice(0, MAX_READ_CHARACTERS),
|
||||
truncated: content.length > MAX_READ_CHARACTERS,
|
||||
};
|
||||
}
|
||||
|
||||
self.addEventListener("message", async (event) => {
|
||||
const { id, type, data } = event.data;
|
||||
const reportProgress = (/** @type {number} */ loaded, /** @type {number} */ total) => {
|
||||
self.postMessage({ id, status: "progress", loaded, total });
|
||||
};
|
||||
|
||||
try {
|
||||
const result = type === "search"
|
||||
? await search(data, reportProgress)
|
||||
: type === "explain"
|
||||
? await explain(data, reportProgress)
|
||||
: await read(data);
|
||||
self.postMessage({ id, status: "complete", data: result });
|
||||
} catch (error) {
|
||||
self.postMessage({ id, status: "error", data: errorMessage(error) });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
const CAMEL_BOUNDARY = /([a-z0-9])([A-Z])/g;
|
||||
const NON_WORD = /[^a-z0-9%]+/g;
|
||||
|
||||
/** @param {unknown} value */
|
||||
export function normalize(value) {
|
||||
return String(value)
|
||||
.replace(CAMEL_BOUNDARY, "$1 $2")
|
||||
.toLowerCase()
|
||||
.replace(NON_WORD, " ")
|
||||
.trim()
|
||||
.replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
/** @param {unknown} value */
|
||||
export function tokens(value) {
|
||||
return normalize(value).split(" ").filter((token) => token.length > 1);
|
||||
}
|
||||
|
||||
/** @param {unknown} value */
|
||||
function trigrams(value) {
|
||||
const padded = ` ${normalize(value)} `;
|
||||
const values = new Set();
|
||||
for (let index = 0; index < padded.length - 2; index += 1) {
|
||||
values.add(padded.slice(index, index + 3));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/** @param {unknown} left @param {unknown} right */
|
||||
export function similarity(left, right) {
|
||||
const a = trigrams(left);
|
||||
const b = trigrams(right);
|
||||
if (!a.size || !b.size) return 0;
|
||||
|
||||
let overlap = 0;
|
||||
for (const value of a) if (b.has(value)) overlap += 1;
|
||||
return (2 * overlap) / (a.size + b.size);
|
||||
}
|
||||
|
||||
/** @param {unknown} query @param {unknown} document @param {number} [boost] */
|
||||
export function relevance(query, document, boost = 0) {
|
||||
const needle = normalize(query);
|
||||
const haystack = normalize(document);
|
||||
if (!needle || !haystack) return 0;
|
||||
|
||||
const words = tokens(needle);
|
||||
const exact = haystack.includes(needle) ? 30 : 0;
|
||||
const coverage = words.length
|
||||
? words.filter((word) => haystack.includes(word)).length / words.length
|
||||
: 0;
|
||||
return boost + exact + coverage * 20 + similarity(needle, haystack) * 10;
|
||||
}
|
||||
Reference in New Issue
Block a user