heatmaps: part 9

website/scripts/options/partial.js

@@ -26,6 +26,10 @@ import { createMiningSection } from "./mining.js";
 import { createCointimeSection } from "./cointime.js";
 import { createInvestingSection } from "./investing.js";
 import { demoHeatmapOption } from "../../src/heatmap/demo.js";
+import {
+  oracleEmaHeatmapOption,
+  oracleRawHeatmapOption,
+} from "../../src/heatmap/oracle.js";
 
 // Re-export types for external consumers
 export * from "./types.js";
@@ -299,7 +303,13 @@ export function createPartialOptions() {
 
     {
       name: "Heatmaps",
-      tree: [demoHeatmapOption],
+      tree: [
+        demoHeatmapOption,
+        {
+          name: "Oracle",
+          tree: [oracleRawHeatmapOption, oracleEmaHeatmapOption],
+        },
+      ],
     },
 
     {

website/src/heatmap/demo.js

@@ -4,6 +4,7 @@
 import { createAverageGrid } from "./grid.js";
 import { INFERNO_LUT, intensityColor } from "./lut.js";
 import { GENESIS_DATE, todayISODate } from "./time.js";
+import { defaultTooltip } from "./tooltip.js";
 
 const ROWS = 160;
 const DAY_MS = 86_400_000;
@@ -19,6 +20,7 @@ export const demoHeatmapOption = {
   },
   grid: createAverageGrid({ yStart: 0, yEnd: 1, nativeRows: ROWS }),
   color: intensityColor({ light: INFERNO_LUT, dark: INFERNO_LUT }),
+  tooltip: defaultTooltip,
 };
 
 /**

website/src/heatmap/grid.js

@@ -1,14 +1,5 @@
 /** @import { HeatmapGrid, HeatmapGridFactory, HeatmapRange } from "./types.js" */
 
-/**
- * @param {number} value
- * @param {number} min
- * @param {number} max
- */
-function clamp(value, min, max) {
-  return Math.min(max, Math.max(min, value));
-}
-
 /**
  * Generic date/y binning with average merge semantics.
  *
@@ -43,12 +34,19 @@ export function createAverageGrid({
       );
       const sums = new Float64Array(cols * rows);
       const counts = new Uint32Array(cols * rows);
+      const maxByCol = new Float64Array(cols);
+      const cumulativeMaxByCol = new Float64Array(cols);
+      let cumulativeMaxDirty = true;
       const ySpan = yEnd - yStart;
 
       /** @param {number} dateIndex */
       function toCol(dateIndex) {
         if (dateIndex < 0 || dateIndex >= dates.length) return undefined;
-        return clamp(Math.floor((dateIndex * cols) / dates.length), 0, cols - 1);
+        return clamp(
+          Math.floor((dateIndex * cols) / dates.length),
+          0,
+          cols - 1,
+        );
       }
 
       /** @param {number} y */
@@ -62,18 +60,19 @@ export function createAverageGrid({
       }
 
       /**
-       * @param {number} dateIndex
+       * @param {number} col
        * @param {number} y
        * @param {number} value
        */
-      function addValue(dateIndex, y, value) {
+      function addValue(col, y, value) {
         if (!Number.isFinite(value)) return undefined;
-        const col = toCol(dateIndex);
         const row = toRow(y);
-        if (col === undefined || row === undefined) return undefined;
+        if (row === undefined) return undefined;
         const index = row * cols + col;
         sums[index] += value;
         counts[index] += 1;
+        maxByCol[col] = Math.max(maxByCol[col], sums[index] / counts[index]);
+        cumulativeMaxDirty = true;
         return col;
       }
 
@@ -83,24 +82,27 @@ export function createAverageGrid({
         cols,
         rows,
         add(dateIndex, points) {
-          let dirty;
+          const col = toCol(dateIndex);
+          if (col === undefined) return undefined;
+          let dirty = false;
           if (points.kind === "implicit") {
             for (let i = 0; i < points.values.length; i++) {
-              const col = addValue(
+              dirty =
-                dateIndex,
+                addValue(
-                points.yStart + i * points.yStep,
+                  col,
-                points.values[i],
+                  points.yStart + i * points.yStep,
-              );
+                  points.values[i],
-              dirty = col ?? dirty;
+                ) !== undefined || dirty;
             }
           } else {
             const length = Math.min(points.y.length, points.values.length);
             for (let i = 0; i < length; i++) {
-              const col = addValue(dateIndex, points.y[i], points.values[i]);
+              dirty =
-              dirty = col ?? dirty;
+                addValue(col, points.y[i], points.values[i]) !== undefined ||
+                dirty;
             }
           }
-          return dirty;
+          return dirty ? col : undefined;
         },
         getValue(col, row) {
           if (col < 0 || col >= cols || row < 0 || row >= rows) {
@@ -109,6 +111,17 @@ export function createAverageGrid({
           const index = row * cols + col;
           return counts[index] ? sums[index] / counts[index] : Number.NaN;
         },
+        getMaxValue(col = cols - 1) {
+          if (cumulativeMaxDirty) {
+            let max = 0;
+            for (let c = 0; c < cols; c++) {
+              max = Math.max(max, maxByCol[c]);
+              cumulativeMaxByCol[c] = max;
+            }
+            cumulativeMaxDirty = false;
+          }
+          return cumulativeMaxByCol[clamp(col, 0, cols - 1)] ?? 0;
+        },
         getDateIndexRange(col) {
           if (col < 0 || col >= cols || dates.length === 0) {
             return emptyRange();
@@ -130,6 +143,15 @@ export function createAverageGrid({
   };
 }
 
+/**
+ * @param {number} value
+ * @param {number} min
+ * @param {number} max
+ */
+function clamp(value, min, max) {
+  return Math.min(max, Math.max(min, value));
+}
+
 /** @returns {HeatmapRange} */
 function emptyRange() {
   return { start: Number.NaN, end: Number.NaN };

website/src/heatmap/index.js

@@ -15,6 +15,8 @@ import { dateRange, GENESIS_DATE, todayISODate, toISODate } from "./time.js";
  */
 
 const MAX_PARALLEL_FETCHES = 8;
+const DEBUG = true;
+const DEBUG_STARTED_AT = performance.now();
 
 /** @type {ReturnType<typeof createRenderer> | undefined} */
 let renderer;
@@ -34,16 +36,29 @@ let currentDates = [];
 let pointsByDate = new Map();
 /** @type {AbortController | undefined} */
 let abortController;
+const dirtyCols = new Set();
 let loadGeneration = 0;
+let paintScheduled = false;
 let initialized = false;
-let from = GENESIS_DATE;
+let from = yearStartISODate(new Date().getUTCFullYear());
 let to = todayISODate();
 
+/**
+ * @param {string} message
+ * @param {Record<string, unknown>} [data]
+ */
+function debug(message, data) {
+  if (!DEBUG) return;
+  const elapsed = Math.round(performance.now() - DEBUG_STARTED_AT);
+  console.log(`[heatmap +${elapsed}ms] ${message}`, data ?? "");
+}
+
 /**
  * Initializes the heatmap pane once for the app lifetime.
  */
 export function init() {
   if (initialized) return;
+  debug("init:start");
   initialized = true;
 
   const header = createHeader();
@@ -64,13 +79,17 @@ export function init() {
 
   new ResizeObserver(
     debounce(() => {
+      debug("resize");
       resizeAndRebuild();
     }, 250),
   ).observe(heatmapElement);
+
+  debug("init:done");
 }
 
 /** @param {HeatmapOption} option */
 export function setOption(option) {
+  debug("setOption", { title: option.title, same: currentOption === option });
   init();
   if (currentOption !== option) {
     currentOption = option;
@@ -84,11 +103,19 @@ export function setOption(option) {
 function resizeAndRebuild() {
   if (!canvas || !renderer) return;
   const { width, height } = canvas.getBoundingClientRect();
+  debug("resizeAndRebuild", { width, height });
   if (renderer.resize(width, height)) rebuildGrid();
 }
 
 function loadRange() {
   if (!currentOption) return;
+  const startedAt = performance.now();
+  debug("loadRange:start", {
+    title: currentOption.title,
+    from,
+    to,
+    cacheSize: pointsByDate.size,
+  });
 
   abortController?.abort();
   const generation = ++loadGeneration;
@@ -96,8 +123,7 @@ function loadRange() {
   const controller = new AbortController();
   abortController = controller;
   currentDates = dateRange(from, to);
-
+  debug("loadRange:dates", { count: currentDates.length });
-  rebuildGrid();
 
   /** @type {{ date: string, dateIndex: number }[]} */
   const missing = [];
@@ -108,21 +134,41 @@ function loadRange() {
   let completed = currentDates.length - missing.length;
   let failed = 0;
   updateStatus(completed, currentDates.length, failed);
+  debug("loadRange:missing", {
+    missing: missing.length,
+    cached: completed,
+    total: currentDates.length,
+  });
 
   if (!missing.length) {
+    debug("loadRange:all-cached:rebuild:start");
+    rebuildGrid();
+    debug("loadRange:all-cached:rebuild:done", {
+      elapsed: Math.round(performance.now() - startedAt),
+    });
     abortController = undefined;
     return;
   }
 
   let cursor = 0;
+  debug("loadRange:workers:start", {
+    workers: Math.min(MAX_PARALLEL_FETCHES, missing.length),
+  });
   const workers = Array.from({
     length: Math.min(MAX_PARALLEL_FETCHES, missing.length),
-  }).map(async () => {
+  }).map(async (_, workerId) => {
+    debug("worker:start", { workerId });
     let index = nextMissingIndex();
     while (index !== undefined) {
       const entry = missing[index];
       try {
+        if (completed < 10) {
+          debug("worker:fetch:start", { workerId, date: entry.date });
+        }
         const points = await option.points.fetch(entry.date, controller.signal);
+        if (completed < 10) {
+          debug("worker:fetch:done", { workerId, date: entry.date });
+        }
         if (isCurrentLoad(option, controller, generation)) {
           pointsByDate.set(entry.date, points);
           addDateToGrid(entry.dateIndex, points);
@@ -135,15 +181,42 @@ function loadRange() {
         if (isCurrentLoad(option, controller, generation)) {
           completed += 1;
           updateStatus(completed, currentDates.length, failed);
+          if (completed <= 10 || completed % 25 === 0 || completed === currentDates.length) {
+            debug("loadRange:progress", {
+              completed,
+              total: currentDates.length,
+              failed,
+              elapsed: Math.round(performance.now() - startedAt),
+            });
+          }
         }
       }
       index = nextMissingIndex();
     }
+    debug("worker:done", { workerId });
+  });
+
+  debug("loadRange:rebuild:start");
+  rebuildGrid();
+  debug("loadRange:rebuild:done", {
+    elapsed: Math.round(performance.now() - startedAt),
   });
 
   void Promise.all(workers).then(() => {
     if (isCurrentLoad(option, controller, generation)) {
       updateStatus(completed, currentDates.length, failed);
+      debug("loadRange:final-paint:start", {
+        completed,
+        total: currentDates.length,
+        failed,
+      });
+      paint();
+      debug("loadRange:done", {
+        completed,
+        total: currentDates.length,
+        failed,
+        elapsed: Math.round(performance.now() - startedAt),
+      });
     }
   });
 
@@ -170,6 +243,7 @@ function isCurrentLoad(option, controller, generation) {
 }
 
 function rebuildGrid() {
+  const startedAt = performance.now();
   if (
     !currentOption ||
     !renderer ||
@@ -178,21 +252,39 @@ function rebuildGrid() {
     !currentDates.length
   ) {
     currentGrid = undefined;
+    debug("rebuildGrid:skip");
     return;
   }
 
+  debug("rebuildGrid:create:start", {
+    dates: currentDates.length,
+    width: renderer.width,
+    height: renderer.height,
+    cached: pointsByDate.size,
+  });
   currentGrid = currentOption.grid.create({
     dates: currentDates,
     width: renderer.width,
     height: renderer.height,
   });
 
+  let added = 0;
   for (let i = 0; i < currentDates.length; i++) {
     const points = pointsByDate.get(currentDates[i]);
-    if (points) currentGrid.add(i, points);
+    if (points) {
+      currentGrid.add(i, points);
+      added += 1;
+    }
   }
 
+  debug("rebuildGrid:add:done", {
+    added,
+    elapsed: Math.round(performance.now() - startedAt),
+  });
   paint();
+  debug("rebuildGrid:paint:done", {
+    elapsed: Math.round(performance.now() - startedAt),
+  });
 }
 
 /**
@@ -202,7 +294,22 @@ function rebuildGrid() {
 function addDateToGrid(dateIndex, points) {
   if (!currentGrid) return;
   const dirtyCol = currentGrid.add(dateIndex, points);
-  if (dirtyCol !== undefined) paint([dirtyCol]);
+  if (dirtyCol !== undefined) schedulePaint(dirtyCol);
+}
+
+/** @param {number} col */
+function schedulePaint(col) {
+  if (dirtyCols.size === 0) debug("paint:schedule", { col });
+  dirtyCols.add(col);
+  if (paintScheduled) return;
+  paintScheduled = true;
+  requestAnimationFrame(() => {
+    paintScheduled = false;
+    if (!dirtyCols.size) return;
+    const cols = Array.from(dirtyCols);
+    dirtyCols.clear();
+    paint(cols);
+  });
 }
 
 /** @param {Iterable<number>} [dirty] */
@@ -256,7 +363,7 @@ function createRangeControls() {
   const currentYear = new Date().getUTCFullYear();
   const fromChoices = createFromChoices(currentYear);
   const toChoices = createToChoices(currentYear);
-  let fromChoice = fromChoices[0];
+  let fromChoice = fromChoices.at(-1) ?? fromChoices[0];
   let toChoice = toChoices[0];
 
   const fromSelect = createSelect({

website/src/heatmap/lut.js

@@ -29,6 +29,24 @@ export function intensityColor({ light, dark }) {
   };
 }
 
+/**
+ * @param {Object} args
+ * @param {ArrayLike<number>} args.light
+ * @param {ArrayLike<number>} args.dark
+ * @returns {HeatmapColorFn}
+ */
+export function logIntensityColor({ light, dark }) {
+  return (value, context) => {
+    if (!Number.isFinite(value) || value <= 0) return 0x00000000;
+    const max = context.grid.getMaxValue(context.col);
+    if (max <= 0) return 0x00000000;
+    const lut = context.dark ? dark : light;
+    const t = Math.log2(value + 1) / Math.log2(max + 1);
+    const index = Math.min(255, Math.max(0, Math.round(t * 255)));
+    return lut[index] ?? 0x00000000;
+  };
+}
+
 /**
  * @param {number[][]} stops - Tuples of [position, red, green, blue].
  */

website/src/heatmap/oracle.js

@@ -0,0 +1,104 @@
+/** @import { PartialHeatmapOption } from "../../scripts/options/types.js" */
+/** @import { HeatmapPoints } from "./types.js" */
+
+import { brk } from "../../scripts/utils/client.js";
+import { createAverageGrid } from "./grid.js";
+import { INFERNO_LUT, logIntensityColor } from "./lut.js";
+import { defaultTooltip } from "./tooltip.js";
+
+const BINS = 2400;
+const MIN_LOG = -8;
+const BINS_PER_DECADE = 200;
+const DEBUG = true;
+const DEBUG_STARTED_AT = performance.now();
+let fetchLogCount = 0;
+
+/**
+ * @param {string} message
+ * @param {Record<string, unknown>} [data]
+ */
+function debug(message, data) {
+  if (!DEBUG) return;
+  const elapsed = Math.round(performance.now() - DEBUG_STARTED_AT);
+  console.log(`[heatmap:oracle +${elapsed}ms] ${message}`, data ?? "");
+}
+
+export const oracleRawHeatmapOption = createOracleHeatmapOption("raw", "Raw");
+export const oracleEmaHeatmapOption = createOracleHeatmapOption("ema", "EMA");
+
+/**
+ * @param {"raw" | "ema"} mode
+ * @param {string} name
+ * @returns {PartialHeatmapOption}
+ */
+function createOracleHeatmapOption(mode, name) {
+  return {
+    kind: "heatmap",
+    name,
+    title: `Oracle ${name} Histogram`,
+    points: {
+      fetch: (date, signal) => fetchOraclePoints(mode, date, signal),
+    },
+    grid: createAverageGrid({
+      yStart: MIN_LOG,
+      yEnd: MIN_LOG + BINS / BINS_PER_DECADE,
+      nativeRows: BINS,
+    }),
+    color: logIntensityColor({ light: INFERNO_LUT, dark: INFERNO_LUT }),
+    tooltip: defaultTooltip,
+  };
+}
+
+/**
+ * @param {"raw" | "ema"} mode
+ * @param {string} date
+ * @param {AbortSignal} signal
+ * @returns {Promise<HeatmapPoints>}
+ */
+async function fetchOraclePoints(mode, date, signal) {
+  const shouldLog = DEBUG && fetchLogCount < 20;
+  fetchLogCount += 1;
+  const startedAt = performance.now();
+  if (shouldLog) debug("fetch:start", { mode, date });
+  const values = await firstAvailable((onValue) =>
+    mode === "raw"
+      ? brk.getOracleHistogramRaw(date, { signal, onValue })
+      : brk.getOracleHistogramEma(date, { signal, onValue }),
+  );
+  if (shouldLog) {
+    debug("fetch:done", {
+      mode,
+      date,
+      length: values.length,
+      elapsed: Math.round(performance.now() - startedAt),
+    });
+  }
+
+  return {
+    kind: "implicit",
+    yStart: MIN_LOG,
+    yStep: 1 / BINS_PER_DECADE,
+    values,
+  };
+}
+
+/**
+ * @param {(onValue: (value: number[]) => void) => Promise<number[]>} fetch
+ * @returns {Promise<number[]>}
+ */
+function firstAvailable(fetch) {
+  return new Promise((resolve, reject) => {
+    let settled = false;
+
+    /** @param {number[]} value */
+    const resolveOnce = (value) => {
+      if (settled) return;
+      settled = true;
+      resolve(value);
+    };
+
+    fetch(resolveOnce).then(resolveOnce, (error) => {
+      if (!settled) reject(error);
+    });
+  });
+}

website/src/heatmap/tooltip.js

@@ -0,0 +1,25 @@
+/** @import { HeatmapTooltipFn } from "./types.js" */
+
+import { numberToShortUSFormat } from "../../scripts/utils/format.js";
+
+/** @satisfies {HeatmapTooltipFn} */
+export const defaultTooltip = ({ grid, col, row }) => {
+  const dateRange = grid.getDateIndexRange(col);
+  const yRange = grid.getYRange(row);
+  const value = grid.getValue(col, row);
+
+  const from = grid.dates[dateRange.start] ?? "";
+  const to = grid.dates[dateRange.end] ?? from;
+  const date = from === to ? from : `${from} to ${to}`;
+
+  return [
+    date,
+    `y ${formatNumber(yRange.start)} to ${formatNumber(yRange.end)}`,
+    `value ${formatNumber(value)}`,
+  ].join("\n");
+};
+
+/** @param {number} value */
+function formatNumber(value) {
+  return numberToShortUSFormat(value);
+}

website/src/heatmap/types.js

@@ -25,6 +25,7 @@
  * @property {number} rows
  * @property {(dateIndex: number, points: HeatmapPoints) => number | undefined} add
  * @property {(col: number, row: number) => number} getValue
+ * @property {(col?: number) => number} getMaxValue
  * @property {(col: number) => HeatmapRange} getDateIndexRange
  * @property {(row: number) => HeatmapRange} getYRange
  *