next: ai part 3

website_next/ask/tools/source/formula.js

@@ -0,0 +1,167 @@
+const DEFINITION = /\b([A-Za-z][A-Za-z0-9_]*)\s*=\s*(.+)$/;
+const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_]*$/;
+
+/** @param {string} value */
+function cleanExpression(value) {
+  return value
+    .replace(/[`.;]+$/g, "")
+    .replace(/[·×]/g, "*")
+    .replace(/([A-Za-z][A-Za-z0-9_]*)²/g, "$1^2")
+    .replace(/\s+/g, "")
+    .replace(/^sum/i, "Σ");
+}
+
+/** @param {string} value @param {string} operator */
+function splitTopLevel(value, operator) {
+  let depth = 0;
+
+  for (let index = 0; index < value.length; index += 1) {
+    const character = value[index];
+    if (character === "(") depth += 1;
+    else if (character === ")") depth -= 1;
+    else if (character === operator && depth === 0) {
+      return [value.slice(0, index), value.slice(index + 1)];
+    }
+  }
+
+  return undefined;
+}
+
+/** @param {string} value */
+function unwrapSum(value) {
+  return value.match(/^Σ\((.+)\)$/)?.[1];
+}
+
+/** @param {string} value */
+function factorMap(value) {
+  /** @type {Map<string, number>} */
+  const factors = new Map();
+
+  for (const rawFactor of value.split("*")) {
+    const match = rawFactor.match(/^([A-Za-z][A-Za-z0-9_]*)(?:\^(\d+))?$/);
+    if (!match) return undefined;
+
+    const [, name, rawPower] = match;
+    const power = Number(rawPower ?? 1);
+    if (!IDENTIFIER.test(name) || power < 1) return undefined;
+    factors.set(name, (factors.get(name) ?? 0) + power);
+  }
+
+  return factors;
+}
+
+/** @param {Map<string, number>} total @param {Map<string, number>} part */
+function subtractFactors(total, part) {
+  const result = new Map(total);
+
+  for (const [name, power] of part) {
+    const remaining = (result.get(name) ?? 0) - power;
+    if (remaining < 0) return undefined;
+    if (remaining === 0) result.delete(name);
+    else result.set(name, remaining);
+  }
+
+  return result;
+}
+
+/** @param {Map<string, number>} factors */
+function factorsText(factors) {
+  return [...factors]
+    .flatMap(([name, power]) => Array.from({ length: power }, () => name))
+    .join(" × ");
+}
+
+/** @param {string} value */
+function title(value) {
+  const words = value.replace(/_/g, " ");
+  return words[0].toUpperCase() + words.slice(1);
+}
+
+/** @param {string} value */
+function plural(value) {
+  if (value.includes(" × ")) return `${value} values`;
+  return value.endsWith("s") ? value : `${value}s`;
+}
+
+/** @param {string} metric @param {string} rawExpression */
+function deriveFormula(metric, rawExpression) {
+  const expression = cleanExpression(rawExpression);
+  const division = splitTopLevel(expression, "/");
+  if (!division) return undefined;
+
+  const numeratorExpression = unwrapSum(division[0]);
+  const denominatorExpression = unwrapSum(division[1]);
+  if (!numeratorExpression || !denominatorExpression) return undefined;
+
+  const numerator = factorMap(numeratorExpression);
+  const weight = factorMap(denominatorExpression);
+  if (!numerator || !weight) return undefined;
+
+  const value = subtractFactors(numerator, weight);
+  if (!value?.size) return undefined;
+
+  const otherWeight = subtractFactors(weight, value);
+  return {
+    metric,
+    formula: `${metric} = ${rawExpression.trim()}`,
+    value: factorsText(value),
+    weight: factorsText(weight),
+    otherWeight: otherWeight ? factorsText(otherWeight) : "",
+  };
+}
+
+/** @param {{ path: string, text: string }} file */
+function formulasInFile(file) {
+  return file.text.split("\n").flatMap((line, index) => {
+    const comment = line.replace(/^\s*(?:\/\/\/?|#)\s?/, "").trim();
+    const definition = comment.match(DEFINITION);
+    if (!definition) return [];
+
+    const fact = deriveFormula(definition[1], definition[2]);
+    return fact ? [{ ...fact, path: file.path, line: index + 1 }] : [];
+  });
+}
+
+/** @param {string} question @param {string} metric */
+function questionScore(question, metric) {
+  const normalizedQuestion = question.toLowerCase().replace(/[_-]+/g, " ");
+  const normalizedMetric = metric.toLowerCase().replace(/_/g, " ");
+  if (normalizedQuestion.includes(normalizedMetric)) return normalizedMetric.length + 10;
+
+  const words = normalizedMetric.split(" ");
+  return words.every((word) => normalizedQuestion.includes(word)) ? words.length : 0;
+}
+
+/** @param {{ path: string, text: string }[]} files */
+export function createFormulaIndex(files) {
+  return files.flatMap(formulasInFile);
+}
+
+/**
+ * @param {string} question
+ * @param {ReturnType<typeof createFormulaIndex>} formulas
+ */
+export function explainFormula(question, formulas) {
+  const fact = formulas
+    .map((candidate) => ({
+      candidate,
+      score: questionScore(question, candidate.metric),
+    }))
+    .filter(({ score }) => score > 0)
+    .sort((a, b) => b.score - a.score)[0]?.candidate;
+  if (!fact) return undefined;
+
+  const metric = title(fact.metric);
+  const values = plural(fact.value);
+  const consequence = fact.otherWeight
+    ? ` For the same ${fact.otherWeight}, higher ${values} have more influence on the result.`
+    : "";
+
+  return {
+    answer:
+      `${metric} is a weighted average of ${values}. ` +
+      `Each ${fact.value} is weighted by \`${fact.weight}\`.${consequence}\n\n` +
+      `Source: \`${fact.path}:${fact.line}\``,
+    fact,
+  };
+}

website_next/ask/tools/source/index.js

@@ -0,0 +1,91 @@
+const WORKER_URL = import.meta.resolve("./worker.js");
+
+export class AskSource {
+  /** @type {Worker | undefined} */
+  #worker;
+
+  /** @type {Map<string, { resolve: (value: any) => void, reject: (error: Error) => void, onProgress?: (progress: { loaded: number, total: number }) => void }>} */
+  #pending = new Map();
+
+  /**
+   * @param {string} query
+   * @param {string | undefined} path
+   * @param {(progress: { loaded: number, total: number }) => void} onProgress
+   */
+  search(query, path, onProgress) {
+    return this.#request("search", { query, path }, onProgress);
+  }
+
+  /**
+   * @param {string} question
+   * @param {(progress: { loaded: number, total: number }) => void} onProgress
+   */
+  explain(question, onProgress) {
+    return this.#request("explain", { question }, onProgress);
+  }
+
+  /**
+   * @param {string} path
+   * @param {number} startLine
+   * @param {number} endLine
+   */
+  read(path, startLine, endLine) {
+    return this.#request("read", { path, startLine, endLine });
+  }
+
+  /**
+   * @param {"search" | "read" | "explain"} type
+   * @param {Record<string, unknown>} data
+   * @param {((progress: { loaded: number, total: number }) => void) | undefined} [onProgress]
+   */
+  #request(type, data, onProgress) {
+    this.#ensureWorker();
+    const id = crypto.randomUUID();
+
+    return new Promise((resolve, reject) => {
+      this.#pending.set(id, { resolve, reject, onProgress });
+      this.#worker?.postMessage({ id, type, data });
+    });
+  }
+
+  #ensureWorker() {
+    if (this.#worker) return;
+
+    this.#worker = new Worker(WORKER_URL, { type: "module" });
+    this.#worker.addEventListener("message", this.#handleMessage);
+    this.#worker.addEventListener("error", this.#handleWorkerError);
+  }
+
+  terminate() {
+    const error = new Error("Source search stopped");
+    for (const request of this.#pending.values()) request.reject(error);
+    this.#pending.clear();
+    this.#worker?.terminate();
+    this.#worker = undefined;
+  }
+
+  /** @param {MessageEvent} event */
+  #handleMessage = (event) => {
+    const message = event.data;
+    const request = this.#pending.get(message.id);
+    if (!request) return;
+
+    if (message.status === "progress") {
+      request.onProgress?.({ loaded: message.loaded, total: message.total });
+      return;
+    }
+
+    this.#pending.delete(message.id);
+    if (message.status === "complete") request.resolve(message.data);
+    else request.reject(new Error(message.data));
+  };
+
+  /** @param {ErrorEvent} event */
+  #handleWorkerError = (event) => {
+    const error = new Error(event.message || "The source worker failed");
+    for (const request of this.#pending.values()) request.reject(error);
+    this.#pending.clear();
+    this.#worker?.terminate();
+    this.#worker = undefined;
+  };
+}

website_next/ask/tools/source/worker.js

@@ -0,0 +1,353 @@
+import { createFormulaIndex, explainFormula } from "./formula.js";
+
+const REPOSITORY = "bitcoinresearchkit/brk";
+const TREE_URL = `https://api.github.com/repos/${REPOSITORY}/git/trees/main?recursive=1`;
+const RAW_URL = `https://raw.githubusercontent.com/${REPOSITORY}`;
+const DATABASE_NAME = "bitview-ask-source-v1";
+const STORE_NAME = "snapshots";
+const MAX_FILE_SIZE = 128_000;
+const MAX_READ_LINES = 120;
+const MAX_READ_CHARACTERS = 2_500;
+const MAX_EXCERPT_CHARACTERS = 500;
+const CONCURRENCY = 12;
+const SEARCH_STOPWORDS = new Set([
+  "a",
+  "about",
+  "and",
+  "bitview",
+  "bitcoin",
+  "brk",
+  "code",
+  "define",
+  "does",
+  "explain",
+  "for",
+  "how",
+  "in",
+  "is",
+  "mean",
+  "of",
+  "repo",
+  "repository",
+  "source",
+  "the",
+  "what",
+  "where",
+  "why",
+  "work",
+  "works",
+]);
+const EXTENSIONS = new Set([
+  "css",
+  "html",
+  "js",
+  "json",
+  "md",
+  "mjs",
+  "py",
+  "rs",
+  "sh",
+  "toml",
+  "ts",
+  "yaml",
+  "yml",
+]);
+const EXCLUDED_PREFIXES = [
+  ".git/",
+  ".github/",
+  "modules/",
+  "packages/brk_client/brk_client/",
+  "target/",
+  "website/assets/",
+  "website_next/modules/",
+];
+const EXCLUDED_FILES = new Set([
+  "docs/CHANGELOG.md",
+  "website/scripts/options/scalar.js",
+]);
+
+/** @type {Promise<{ sha: string, entries: { path: string, size: number }[] }> | undefined} */
+let treePromise;
+
+/** @type {Promise<{ sha: string, files: { path: string, text: string }[] }> | undefined} */
+let snapshotPromise;
+
+/** @type {ReturnType<typeof createFormulaIndex> | undefined} */
+let formulaIndex;
+
+/** @param {unknown} error */
+function errorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/** @param {string} path */
+function extension(path) {
+  return path.slice(path.lastIndexOf(".") + 1).toLowerCase();
+}
+
+/** @param {{ type: string, path: string, size?: number }} item */
+function isSource(item) {
+  return (
+    item.type === "blob" &&
+    Number(item.size ?? 0) <= MAX_FILE_SIZE &&
+    EXTENSIONS.has(extension(item.path)) &&
+    !EXCLUDED_FILES.has(item.path) &&
+    !EXCLUDED_PREFIXES.some((prefix) => item.path.startsWith(prefix))
+  );
+}
+
+async function loadTree() {
+  treePromise ??= fetch(TREE_URL, {
+    headers: { Accept: "application/vnd.github+json" },
+  }).then(async (response) => {
+    if (!response.ok) {
+      throw new Error(`GitHub source tree unavailable (${response.status})`);
+    }
+
+    const data = await response.json();
+    if (data.truncated) throw new Error("GitHub returned a truncated source tree");
+
+    const tree = /** @type {{ type: string, path: string, size?: number }[]} */ (
+      data.tree
+    );
+    return {
+      sha: data.sha,
+      entries: tree.filter(isSource).map((item) => ({
+        path: item.path,
+        size: Number(item.size ?? 0),
+      })),
+    };
+  });
+
+  return treePromise;
+}
+
+/** @param {string} sha @param {string} path */
+async function fetchSource(sha, path) {
+  const encodedPath = path.split("/").map(encodeURIComponent).join("/");
+  const response = await fetch(`${RAW_URL}/${sha}/${encodedPath}`);
+  if (!response.ok) throw new Error(`Could not fetch ${path}`);
+  return response.text();
+}
+
+function openDatabase() {
+  return new Promise((resolve, reject) => {
+    const request = indexedDB.open(DATABASE_NAME, 1);
+    request.addEventListener("upgradeneeded", () => {
+      request.result.createObjectStore(STORE_NAME, { keyPath: "sha" });
+    });
+    request.addEventListener("success", () => resolve(request.result));
+    request.addEventListener("error", () => reject(request.error));
+  });
+}
+
+/** @param {IDBDatabase} database @param {string} sha */
+function readSnapshot(database, sha) {
+  return new Promise((resolve, reject) => {
+    const request = database
+      .transaction(STORE_NAME, "readonly")
+      .objectStore(STORE_NAME)
+      .get(sha);
+    request.addEventListener("success", () => resolve(request.result));
+    request.addEventListener("error", () => reject(request.error));
+  });
+}
+
+/** @param {IDBDatabase} database @param {{ sha: string, files: { path: string, text: string }[] }} snapshot */
+function writeSnapshot(database, snapshot) {
+  return new Promise((resolve, reject) => {
+    const transaction = database.transaction(STORE_NAME, "readwrite");
+    const store = transaction.objectStore(STORE_NAME);
+    store.clear();
+    store.put(snapshot);
+    transaction.addEventListener("complete", () => resolve(undefined));
+    transaction.addEventListener("error", () => reject(transaction.error));
+  });
+}
+
+/** @param {(loaded: number, total: number) => void} reportProgress */
+async function loadSnapshot(reportProgress) {
+  const tree = await loadTree();
+  const database = /** @type {IDBDatabase} */ (await openDatabase());
+
+  try {
+    const cached = /** @type {{ sha: string, files: { path: string, text: string }[] } | undefined} */ (
+      await readSnapshot(database, tree.sha)
+    );
+    if (cached) {
+      reportProgress(cached.files.length, cached.files.length);
+      return cached;
+    }
+
+    const files = new Array(tree.entries.length);
+    let cursor = 0;
+    let loaded = 0;
+    reportProgress(loaded, tree.entries.length);
+
+    async function download() {
+      while (cursor < tree.entries.length) {
+        const index = cursor;
+        cursor += 1;
+        const entry = tree.entries[index];
+        files[index] = {
+          path: entry.path,
+          text: await fetchSource(tree.sha, entry.path),
+        };
+        loaded += 1;
+        if (loaded % 20 === 0 || loaded === tree.entries.length) {
+          reportProgress(loaded, tree.entries.length);
+        }
+      }
+    }
+
+    await Promise.all(
+      Array.from({ length: Math.min(CONCURRENCY, tree.entries.length) }, download),
+    );
+    const snapshot = { sha: tree.sha, files };
+    await writeSnapshot(database, snapshot);
+    return snapshot;
+  } finally {
+    database.close();
+  }
+}
+
+/** @param {(loaded: number, total: number) => void} reportProgress */
+function ensureSnapshot(reportProgress) {
+  snapshotPromise ??= loadSnapshot(reportProgress).catch((error) => {
+    snapshotPromise = undefined;
+    throw error;
+  });
+  return snapshotPromise;
+}
+
+/** @param {string} value */
+function normalize(value) {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/** @param {string} text @param {number} index */
+function lineAt(text, index) {
+  return text.slice(0, index).split("\n").length;
+}
+
+/** @param {string} text @param {number} line */
+function excerptAt(text, line) {
+  const lines = text.split("\n");
+  const start = Math.max(1, line - 3);
+  const end = Math.min(lines.length, line + 3);
+  return {
+    startLine: start,
+    endLine: end,
+    content: lines
+      .slice(start - 1, end)
+      .join("\n")
+      .slice(0, MAX_EXCERPT_CHARACTERS),
+  };
+}
+
+/**
+ * @param {{ query: string, path?: string }} args
+ * @param {(loaded: number, total: number) => void} reportProgress
+ */
+async function search(args, reportProgress) {
+  const rawQuery = normalize(args.query);
+  if (!rawQuery) throw new Error("Search query is empty");
+  const pathPrefix = String(args.path ?? "").replace(/^\/+/, "");
+  const rawTokens = rawQuery.split(" ");
+  const tokens = rawTokens.filter((token) => !SEARCH_STOPWORDS.has(token));
+  const searchTokens = tokens.length ? tokens : rawTokens;
+  const query = searchTokens.join(" ");
+  const snapshot = await ensureSnapshot(reportProgress);
+  const matches = [];
+
+  for (const file of snapshot.files) {
+    if (pathPrefix && !file.path.startsWith(pathPrefix)) continue;
+    const normalized = normalize(file.text);
+    let index = normalized.indexOf(query);
+    let score = 2;
+
+    if (index < 0 && searchTokens.every((token) => normalized.includes(token))) {
+      index = normalized.indexOf(searchTokens[0]);
+      score = 1;
+    }
+    if (index < 0) continue;
+
+    const rawIndex = file.text.toLowerCase().indexOf(searchTokens[0]);
+    const line = lineAt(file.text, Math.max(0, rawIndex));
+    matches.push({
+      path: file.path,
+      score: score + (normalize(file.path).includes(query) ? 2 : 0),
+      ...excerptAt(file.text, line),
+    });
+  }
+
+  matches.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+  return {
+    revision: snapshot.sha,
+    matches: matches.slice(0, 4).map(({ score, ...match }) => match),
+  };
+}
+
+/**
+ * @param {{ question: string }} args
+ * @param {(loaded: number, total: number) => void} reportProgress
+ */
+async function explain(args, reportProgress) {
+  const snapshot = await ensureSnapshot(reportProgress);
+  formulaIndex ??= createFormulaIndex(snapshot.files);
+  const result = explainFormula(String(args.question ?? ""), formulaIndex);
+  return result ? { revision: snapshot.sha, ...result } : undefined;
+}
+
+/** @param {{ path: string, startLine: number, endLine: number }} args */
+async function read(args) {
+  const path = String(args.path).replace(/^\/+/, "");
+  const startLine = Math.max(1, Math.floor(Number(args.startLine)));
+  const requestedEnd = Math.max(startLine, Math.floor(Number(args.endLine)));
+  const endLine = Math.min(requestedEnd, startLine + MAX_READ_LINES - 1);
+  const tree = await loadTree();
+  if (!tree.entries.some((entry) => entry.path === path)) {
+    throw new Error(`Source file not found: ${path}`);
+  }
+
+  const snapshot = await snapshotPromise;
+  const text = snapshot?.files.find((file) => file.path === path)?.text ??
+    (await fetchSource(tree.sha, path));
+  const lines = text.split("\n");
+  if (startLine > lines.length) {
+    throw new Error(`${path} only has ${lines.length} lines`);
+  }
+  const lastLine = Math.min(endLine, lines.length);
+  const content = lines.slice(startLine - 1, lastLine).join("\n");
+
+  return {
+    revision: tree.sha,
+    path,
+    startLine,
+    endLine: lastLine,
+    content: content.slice(0, MAX_READ_CHARACTERS),
+    truncated: content.length > MAX_READ_CHARACTERS,
+  };
+}
+
+self.addEventListener("message", async (event) => {
+  const { id, type, data } = event.data;
+  const reportProgress = (/** @type {number} */ loaded, /** @type {number} */ total) => {
+    self.postMessage({ id, status: "progress", loaded, total });
+  };
+
+  try {
+    const result = type === "search"
+      ? await search(data, reportProgress)
+      : type === "explain"
+        ? await explain(data, reportProgress)
+        : await read(data);
+    self.postMessage({ id, status: "complete", data: result });
+  } catch (error) {
+    self.postMessage({ id, status: "error", data: errorMessage(error) });
+  }
+});