mirror of
https://github.com/bitcoinresearchkit/brk.git
synced 2026-07-24 17:38:09 -07:00
next: ai part 3
This commit is contained in:
nym21
@@ -0,0 +1,143 @@
|
||||
import { brk } from "../../../utils/client.js";
|
||||
|
||||
const WORKER_URL = import.meta.resolve("./worker.js");
|
||||
const FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
||||
|
||||
/** @typedef {{ path: string, name: string, endpoint: string, suggestedUnit?: string, matchedQuery?: string, score?: number }} CatalogMetric */
|
||||
|
||||
/** @param {unknown} value */
|
||||
function isMetric(value) {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const by = /** @type {{ by?: Record<string, unknown> }} */ (value).by;
|
||||
return Boolean(
|
||||
by &&
|
||||
Object.values(by).some(
|
||||
(endpoint) =>
|
||||
endpoint &&
|
||||
typeof endpoint === "object" &&
|
||||
typeof /** @type {{ fetch?: unknown }} */ (endpoint).fetch === "function",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
function normalizePath(path) {
|
||||
const normalized = path
|
||||
.trim()
|
||||
.replace(/^brk\.series\./, "")
|
||||
.replace(/^series\./, "");
|
||||
const keys = normalized.split(".").filter(Boolean);
|
||||
|
||||
if (!keys.length || keys.some((key) => FORBIDDEN_KEYS.has(key))) {
|
||||
throw new Error(`Invalid metric path: ${path}`);
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
/** @param {typeof brk} client @param {string} path */
|
||||
export function resolveMetric(client, path) {
|
||||
const keys = normalizePath(path);
|
||||
/** @type {unknown} */
|
||||
let value = client.series;
|
||||
|
||||
for (const key of keys) {
|
||||
if (!value || typeof value !== "object" || !Object.hasOwn(value, key)) {
|
||||
throw new Error(`Metric not found: ${path}`);
|
||||
}
|
||||
value = /** @type {Record<string, unknown>} */ (value)[key];
|
||||
}
|
||||
|
||||
if (!isMetric(value)) throw new Error(`Metric not found: ${path}`);
|
||||
return /** @type {TimeframeMetric} */ (value);
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
export function createMetric(path) {
|
||||
resolveMetric(brk, path);
|
||||
return (/** @type {typeof brk} */ client) => resolveMetric(client, path);
|
||||
}
|
||||
|
||||
class MetricIndex {
|
||||
/** @type {Worker | undefined} */
|
||||
#worker;
|
||||
|
||||
/** @type {Map<string, { resolve: (value: any) => void, reject: (error: Error) => void, onProgress?: () => void }>} */
|
||||
#pending = new Map();
|
||||
|
||||
/** @param {"search" | "categories" | "byName" | "variants"} type @param {Record<string, unknown>} data @param {(() => void) | undefined} [onProgress] */
|
||||
request(type, data, onProgress) {
|
||||
this.#ensureWorker();
|
||||
const id = crypto.randomUUID();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.#pending.set(id, { resolve, reject, onProgress });
|
||||
this.#worker?.postMessage({ id, type, data });
|
||||
});
|
||||
}
|
||||
|
||||
#ensureWorker() {
|
||||
if (this.#worker) return;
|
||||
this.#worker = new Worker(WORKER_URL, { type: "module" });
|
||||
this.#worker.addEventListener("message", this.#handleMessage);
|
||||
this.#worker.addEventListener("error", this.#handleError);
|
||||
}
|
||||
|
||||
terminate() {
|
||||
const error = new Error("Metric search stopped");
|
||||
for (const request of this.#pending.values()) request.reject(error);
|
||||
this.#pending.clear();
|
||||
this.#worker?.terminate();
|
||||
this.#worker = undefined;
|
||||
}
|
||||
|
||||
/** @param {MessageEvent} event */
|
||||
#handleMessage = (event) => {
|
||||
const message = event.data;
|
||||
const request = this.#pending.get(message.id);
|
||||
if (!request) return;
|
||||
|
||||
if (message.status === "progress") {
|
||||
request.onProgress?.();
|
||||
return;
|
||||
}
|
||||
|
||||
this.#pending.delete(message.id);
|
||||
if (message.status === "complete") request.resolve(message.data);
|
||||
else request.reject(new Error(message.data));
|
||||
};
|
||||
|
||||
/** @param {ErrorEvent} event */
|
||||
#handleError = (event) => {
|
||||
const error = new Error(event.message || "The metric index failed");
|
||||
for (const request of this.#pending.values()) request.reject(error);
|
||||
this.#pending.clear();
|
||||
this.#worker?.terminate();
|
||||
this.#worker = undefined;
|
||||
};
|
||||
}
|
||||
|
||||
const index = new MetricIndex();
|
||||
|
||||
/** @param {string[]} queries @param {number} [limit] @param {string[]} [prefixes] @param {(() => void) | undefined} [onProgress] @returns {Promise<CatalogMetric[]>} */
|
||||
export function searchMetrics(queries, limit = 16, prefixes = [], onProgress) {
|
||||
return index.request("search", { queries, limit, prefixes }, onProgress);
|
||||
}
|
||||
|
||||
/** @returns {Promise<{ path: string, label: string, count: number, examples: string[] }[]>} */
|
||||
export function metricCategories() {
|
||||
return index.request("categories", {});
|
||||
}
|
||||
|
||||
/** @param {string} name @returns {Promise<CatalogMetric | undefined>} */
|
||||
export function metricByName(name) {
|
||||
return index.request("byName", { name });
|
||||
}
|
||||
|
||||
/** @param {{ name: string }} metric @param {string} query @returns {Promise<{ totalSeries: number, groups: { family: string, examples: string[] }[], series: CatalogMetric[] } | undefined>} */
|
||||
export function metricVariants(metric, query = "") {
|
||||
return index.request("variants", { name: metric.name, query });
|
||||
}
|
||||
|
||||
export function terminateMetricIndex() {
|
||||
index.terminate();
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
import { QuickMatch, QuickMatchConfig } from "../../../modules/quickmatch-js/0.5.0/src/index.js";
|
||||
import { brk } from "../../../utils/client.js";
|
||||
|
||||
const SEARCH_CANDIDATES = 1_024;
|
||||
|
||||
/** @typedef {{ path: string, name: string, endpoint: string, document: string }} CatalogMetric */
|
||||
|
||||
/** @param {unknown} value */
|
||||
function isMetric(value) {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const by = /** @type {{ by?: Record<string, unknown> }} */ (value).by;
|
||||
return Boolean(
|
||||
by &&
|
||||
Object.values(by).some(
|
||||
(endpoint) =>
|
||||
endpoint &&
|
||||
typeof endpoint === "object" &&
|
||||
typeof /** @type {{ fetch?: unknown }} */ (endpoint).fetch === "function",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/** @param {string} value */
|
||||
function searchable(value) {
|
||||
return value
|
||||
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
||||
.replace(/[_./|:-]+/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
/** @param {string} path */
|
||||
function suggestedUnit(path) {
|
||||
if (/(percent|ratio|dominance|rate)/i.test(path)) return "percent";
|
||||
if (/(usd|price|cap)/i.test(path)) return "usd";
|
||||
if (/(btc|supply|value)/i.test(path)) return "btc";
|
||||
if (/(address|addr)/i.test(path)) return "addresses";
|
||||
if (/(utxo|output)/i.test(path)) return "utxos";
|
||||
if (/(block|height|epoch)/i.test(path)) return "blocks";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** @param {number} limit */
|
||||
function createConfig(limit = SEARCH_CANDIDATES) {
|
||||
// The model supplies semantic wording, where unmatched words are usually context rather than typos.
|
||||
return new QuickMatchConfig()
|
||||
.withLimit(limit)
|
||||
.withTrigramBudget(0)
|
||||
.withMinScore(2)
|
||||
.withSeparators("_- :/.|");
|
||||
}
|
||||
|
||||
function buildState() {
|
||||
/** @type {CatalogMetric[]} */
|
||||
const items = [];
|
||||
/** @type {Map<string, CatalogMetric>} */
|
||||
const byName = new Map();
|
||||
/** @type {Map<string, CatalogMetric>} */
|
||||
const byDocument = new Map();
|
||||
const seen = new WeakSet();
|
||||
/** @type {{ value: unknown, keys: string[] }[]} */
|
||||
const pending = [{ value: brk.series, keys: [] }];
|
||||
|
||||
while (pending.length) {
|
||||
const { value, keys } = /** @type {{ value: unknown, keys: string[] }} */ (pending.pop());
|
||||
if (!value || typeof value !== "object" || seen.has(value)) continue;
|
||||
seen.add(value);
|
||||
|
||||
if (isMetric(value)) {
|
||||
const raw = /** @type {{ name?: string, by: Record<string, { path?: string }> }} */ (value);
|
||||
const path = keys.join(".");
|
||||
const name = raw.name ?? keys.at(-1) ?? "";
|
||||
const endpoint = Object.values(raw.by).find((item) => item.path)?.path ?? "";
|
||||
const document = searchable(`${name} | ${path} | ${endpoint}`);
|
||||
const metric = { path, name, endpoint, document };
|
||||
items.push(metric);
|
||||
if (!byName.has(name)) byName.set(name, metric);
|
||||
byDocument.set(document, metric);
|
||||
} else {
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (key !== "by") pending.push({ value: child, keys: [...keys, key] });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const config = createConfig();
|
||||
const matcher = new QuickMatch(items.map(({ document }) => document), config);
|
||||
/** @type {Map<string, { matcher: QuickMatch, config: QuickMatchConfig }>} */
|
||||
const scoped = new Map();
|
||||
return { items, byName, byDocument, matcher, config, scoped };
|
||||
}
|
||||
|
||||
/** @type {Promise<ReturnType<typeof buildState>> | undefined} */
|
||||
let statePromise;
|
||||
|
||||
/** @param {string} id */
|
||||
function state(id) {
|
||||
if (!statePromise) {
|
||||
self.postMessage({ id, status: "progress" });
|
||||
statePromise = Promise.resolve().then(buildState);
|
||||
}
|
||||
return statePromise;
|
||||
}
|
||||
|
||||
/** @param {CatalogMetric} metric */
|
||||
function publicMetric(metric) {
|
||||
return {
|
||||
path: metric.path,
|
||||
name: metric.name,
|
||||
endpoint: metric.endpoint,
|
||||
suggestedUnit: suggestedUnit(metric.path),
|
||||
};
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string[]} prefixes */
|
||||
function scopedIndex(index, prefixes) {
|
||||
const inScope = (/** @type {CatalogMetric} */ metric) =>
|
||||
!prefixes.length || prefixes.some((prefix) =>
|
||||
metric.path === prefix || metric.path.startsWith(`${prefix}.`)
|
||||
);
|
||||
if (!prefixes.length) return { matcher: index.matcher, config: index.config, inScope };
|
||||
|
||||
const key = [...prefixes].sort().join("|");
|
||||
let scoped = index.scoped.get(key);
|
||||
if (!scoped) {
|
||||
const config = createConfig();
|
||||
const documents = index.items.filter(inScope).map(({ document }) => document);
|
||||
scoped = { matcher: new QuickMatch(documents, config), config };
|
||||
index.scoped.set(key, scoped);
|
||||
}
|
||||
return { ...scoped, inScope };
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string} query @param {number} limit @param {string[]} prefixes */
|
||||
function searchOne(index, query, limit, prefixes) {
|
||||
const scope = scopedIndex(index, prefixes);
|
||||
const normalizedQuery = searchable(query);
|
||||
const full = scope.matcher.matchesWith(
|
||||
normalizedQuery,
|
||||
scope.config.withLimit(SEARCH_CANDIDATES),
|
||||
);
|
||||
const fullRanks = new Map(full.map((document, rank) => [document, rank]));
|
||||
const scores = new Map();
|
||||
for (const word of [...new Set(normalizedQuery.split(" "))]) {
|
||||
if (!word) continue;
|
||||
const matches = scope.matcher.matchesWith(
|
||||
word,
|
||||
scope.config.withLimit(SEARCH_CANDIDATES),
|
||||
);
|
||||
for (const [rank, document] of matches.entries()) {
|
||||
const score = scores.get(document) ?? { matched: 0, ranks: 0 };
|
||||
score.matched += 1;
|
||||
score.ranks += rank;
|
||||
scores.set(document, score);
|
||||
}
|
||||
}
|
||||
const documents = [...new Set([...full, ...scores.keys()])]
|
||||
.sort((left, right) => {
|
||||
const a = scores.get(left) ?? { matched: 0, ranks: SEARCH_CANDIDATES };
|
||||
const b = scores.get(right) ?? { matched: 0, ranks: SEARCH_CANDIDATES };
|
||||
return b.matched - a.matched ||
|
||||
(fullRanks.get(left) ?? SEARCH_CANDIDATES) -
|
||||
(fullRanks.get(right) ?? SEARCH_CANDIDATES) ||
|
||||
a.ranks - b.ranks ||
|
||||
left.length - right.length ||
|
||||
left.localeCompare(right);
|
||||
});
|
||||
|
||||
return documents
|
||||
.map((document) => index.byDocument.get(document))
|
||||
.filter((metric) => metric && scope.inScope(metric))
|
||||
.slice(0, limit)
|
||||
.map((metric, rank) => ({
|
||||
...publicMetric(/** @type {CatalogMetric} */ (metric)),
|
||||
matchedQuery: query,
|
||||
score: 1_000 - rank,
|
||||
}));
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string[]} queries @param {number} limit @param {string[]} prefixes */
|
||||
function search(index, queries, limit, prefixes) {
|
||||
const groups = queries.map((query) => searchOne(index, query, limit, prefixes));
|
||||
const output = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (let rank = 0; output.length < limit; rank += 1) {
|
||||
let added = false;
|
||||
for (const group of groups) {
|
||||
const metric = group[rank];
|
||||
if (!metric || seen.has(metric.path)) continue;
|
||||
seen.add(metric.path);
|
||||
output.push(metric);
|
||||
added = true;
|
||||
if (output.length === limit) break;
|
||||
}
|
||||
if (!added) break;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index */
|
||||
function categories(index) {
|
||||
/** @type {Map<string, { path: string, label: string, count: number, branches: Map<string, number>[] }>} */
|
||||
const values = new Map();
|
||||
for (const metric of index.items) {
|
||||
const [root, branch] = metric.path.split(".");
|
||||
if (!root) continue;
|
||||
const category = values.get(root) ?? {
|
||||
path: root,
|
||||
label: searchable(root),
|
||||
count: 0,
|
||||
branches: [new Map(), new Map()],
|
||||
};
|
||||
category.count += 1;
|
||||
if (branch) {
|
||||
category.branches[0].set(branch, (category.branches[0].get(branch) ?? 0) + 1);
|
||||
const selector = metric.path.split(".").slice(1, 3).join(" / ");
|
||||
if (selector !== branch) {
|
||||
category.branches[1].set(selector, (category.branches[1].get(selector) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
values.set(root, category);
|
||||
}
|
||||
return [...values.values()]
|
||||
.map(({ branches, ...category }) => ({
|
||||
...category,
|
||||
examples: branches.flatMap((level, index) => [...level]
|
||||
.sort((left, right) => right[1] - left[1])
|
||||
.slice(0, index === 0 ? 4 : 8)
|
||||
.map(([name]) => searchable(name)))
|
||||
.slice(0, 8),
|
||||
}))
|
||||
.sort((left, right) => left.label.localeCompare(right.label));
|
||||
}
|
||||
|
||||
/** @param {ReturnType<typeof buildState>} index @param {string} name @param {string} query */
|
||||
function variants(index, name, query) {
|
||||
const suffix = `_${name}`;
|
||||
const candidates = index.items.filter((candidate) =>
|
||||
candidate.name === name || candidate.name.endsWith(suffix)
|
||||
);
|
||||
if (candidates.length <= 1) return undefined;
|
||||
|
||||
const preferredPaths = new Map(
|
||||
index.matcher.matchesWith(searchable(query), index.config.withLimit(SEARCH_CANDIDATES))
|
||||
.map((document, rank) => [index.byDocument.get(document)?.path, rank]),
|
||||
);
|
||||
const ranked = candidates
|
||||
.map((candidate) => ({
|
||||
...publicMetric(candidate),
|
||||
rank: preferredPaths.get(candidate.path) ?? SEARCH_CANDIDATES,
|
||||
}))
|
||||
.sort((left, right) =>
|
||||
Number(right.name === name) - Number(left.name === name) ||
|
||||
left.rank - right.rank ||
|
||||
left.path.localeCompare(right.path)
|
||||
);
|
||||
|
||||
let commonSuffix = candidates[0].path.split(".");
|
||||
for (const candidate of candidates.slice(1)) {
|
||||
const path = candidate.path.split(".");
|
||||
let count = 0;
|
||||
while (
|
||||
count < commonSuffix.length &&
|
||||
count < path.length &&
|
||||
commonSuffix[commonSuffix.length - 1 - count] === path[path.length - 1 - count]
|
||||
) count += 1;
|
||||
commonSuffix = count ? commonSuffix.slice(-count) : [];
|
||||
}
|
||||
|
||||
const groups = new Map();
|
||||
for (const candidate of ranked) {
|
||||
const selector = candidate.path.split(".").slice(0, -commonSuffix.length);
|
||||
const cohortIndex = selector.indexOf("cohorts");
|
||||
const cohort = selector.slice(cohortIndex < 0 ? 0 : cohortIndex + 1);
|
||||
const family = cohort.length > 2 ? cohort.slice(0, 2).join(" / ") : cohort[0] ?? "root";
|
||||
const value = cohort.length > 2 ? cohort.slice(2).join(" / ") : cohort[1] ?? "all";
|
||||
const group = groups.get(family) ?? { family, count: 0, examples: [] };
|
||||
group.count += 1;
|
||||
if (group.examples.length < 5) group.examples.push(value);
|
||||
groups.set(family, group);
|
||||
}
|
||||
|
||||
return {
|
||||
totalSeries: ranked.length,
|
||||
groups: [...groups.values()].slice(0, 8),
|
||||
series: ranked.slice(0, 16).map(({ path, name: metricName, endpoint, suggestedUnit }) => ({
|
||||
path,
|
||||
name: metricName,
|
||||
endpoint,
|
||||
suggestedUnit,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
self.addEventListener("message", async (event) => {
|
||||
const { id, type, data } = event.data;
|
||||
try {
|
||||
const index = await state(id);
|
||||
let result;
|
||||
if (type === "search") {
|
||||
result = search(index, data.queries, data.limit, data.prefixes);
|
||||
} else if (type === "categories") {
|
||||
result = categories(index);
|
||||
} else if (type === "byName") {
|
||||
const metric = index.byName.get(data.name);
|
||||
result = metric ? publicMetric(metric) : undefined;
|
||||
} else if (type === "variants") {
|
||||
result = variants(index, data.name, data.query);
|
||||
} else {
|
||||
throw new Error(`Unknown metric request: ${type}`);
|
||||
}
|
||||
self.postMessage({ id, status: "complete", data: result });
|
||||
} catch (error) {
|
||||
self.postMessage({
|
||||
id,
|
||||
status: "error",
|
||||
data: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user