heatmaps: part 19

crates/brk_oracle/src/scale.rs

@@ -5,8 +5,8 @@ const MIN_LOG_BTC: i32 = -8;
 const MAX_LOG_BTC: i32 = 4;
 pub const NUM_BINS: usize = BINS_PER_DECADE * (MAX_LOG_BTC - MIN_LOG_BTC) as usize;
 
-/// Per-block round-dollar payment counts, one `u32` per log-scale bin: the
+/// Per-bin integer counts on the oracle log scale: used for both oracle-eligible
-/// oracle's ring-buffer element and the `histogram/raw/*` wire payload.
+/// payment histograms and unfiltered output histograms.
 pub type HistogramRaw = Histogram<u32, NUM_BINS>;
 
 /// Smoothed EMA over the window, one `f64` per bin. The stencil search reads it,

crates/brk_query/src/impl/oracle.rs

@@ -30,9 +30,10 @@ impl Query {
     }
 
     /// Smoothed payment output histogram for a calendar `day`: the bin-by-bin average of
-    /// every confirmed block's per-block EMA. Each block's EMA is reconstructed
+    /// every confirmed block's per-block EMA. The first block in each EMA config
-    /// independently (seed-independent, so exact); averaging keeps the result an
+    /// segment is reconstructed exactly, then later blocks in the segment are walked
-    /// intensive per-block rate rather than letting a busy day dominate.
+    /// sequentially. Averaging keeps the result an intensive per-block rate rather
+    /// than letting a busy day dominate.
     pub fn confirmed_payment_histogram_day(&self, day: Day1) -> Result<HistogramEmaCompact> {
         let safe = self.safe_lengths();
         let range = self.day_block_range(day, &safe)?;
@@ -177,18 +178,21 @@ impl Query {
         Ok(cents_to_bin(cents.inner() as f64))
     }
 
-    /// `height < min(spot price len, safe height)` or 404.
+    fn histogram_bound(&self, safe: &Lengths) -> usize {
-    /// Returns the safe lengths so callers cap reads at the same bound.
+        self.computer()
-    fn check_histogram_height(&self, height: usize) -> Result<Lengths> {
-        let safe = self.safe_lengths();
-        let bound = self
-            .computer()
             .prices
             .spot
             .cents
             .height
             .len()
-            .min(safe.height.to_usize());
+            .min(safe.height.to_usize())
+    }
+
+    /// `height < min(spot price len, safe height)` or 404.
+    /// Returns the safe lengths so callers cap reads at the same bound.
+    fn check_histogram_height(&self, height: usize) -> Result<Lengths> {
+        let safe = self.safe_lengths();
+        let bound = self.histogram_bound(&safe);
         if height >= bound {
             return Err(Error::NotFound(format!(
                 "oracle histogram unavailable for height {height}"
@@ -202,14 +206,7 @@ impl Query {
     /// the day has no committed blocks in range.
     fn day_block_range(&self, day: Day1, safe: &Lengths) -> Result<Range<usize>> {
         let first_height = &self.computer().indexes.day1.first_height;
-        let bound = self
+        let bound = self.histogram_bound(safe);
-            .computer()
-            .prices
-            .spot
-            .cents
-            .height
-            .len()
-            .min(safe.height.to_usize());
         let start = first_height
             .collect_one(day)
             .map_or(usize::MAX, |h| h.to_usize());

website/scripts/_types.js

@@ -12,9 +12,9 @@
  *
  * @import { Color } from "./utils/colors.js"
  *
- * @import { HeatmapAxis, HeatmapDefaults, HeatmapPointSource, HeatmapGridFactory, HeatmapColorFn, HeatmapTooltipFn } from "../src/heatmap/types.js"
+ * @import { HeatmapAxis, HeatmapAxisChoice, HeatmapDefaults, HeatmapGrid, HeatmapGridFactory, HeatmapPoints, HeatmapRange, HeatmapPointSource, HeatmapColorFn, HeatmapTooltipFn } from "../src/heatmap/types.js"
  *
- * @import { Option, PartialChartOption, ChartOption, AnyPartialOption, ProcessedOptionAddons, OptionsTree, AnySeriesBlueprint, SeriesType, AnyFetchedSeriesBlueprint, ExplorerOption, UrlOption, PartialOptionsGroup, OptionsGroup, PartialOptionsTree, UtxoCohortObject, AddrCohortObject, CohortObject, CohortGroupObject, FetchedLineSeriesBlueprint, FetchedBaselineSeriesBlueprint, FetchedHistogramSeriesBlueprint, FetchedDotsBaselineSeriesBlueprint, PatternAll, PatternFull, PatternWithAdjusted, PatternWithPercentiles, PatternBasic, PatternBasicWithMarketCap, PatternBasicWithoutMarketCap, PatternWithoutRelative, CohortAll, CohortFull, CohortWithAdjusted, CohortWithPercentiles, CohortBasic, CohortBasicWithMarketCap, CohortBasicWithoutMarketCap, CohortWithoutRelative, CohortAddr, CohortLongTerm, CohortAgeRange, CohortAgeRangeWithMatured, CohortGroupFull, CohortGroupWithAdjusted, CohortGroupWithPercentiles, CohortGroupLongTerm, CohortGroupAgeRange, CohortGroupBasic, CohortGroupBasicWithMarketCap, CohortGroupBasicWithoutMarketCap, CohortGroupWithoutRelative, CohortGroupAddr, UtxoCohortGroupObject, AddrCohortGroupObject, FetchedDotsSeriesBlueprint, HeatmapOption, FetchedCandlestickSeriesBlueprint, FetchedPriceSeriesBlueprint, AnyPricePattern, AnyValuePattern } from "./options/partial.js"
+ * @import { Option, PartialChartOption, ChartOption, AnyPartialOption, ProcessedOptionAddons, OptionsTree, AnySeriesBlueprint, SeriesType, AnyFetchedSeriesBlueprint, ExplorerOption, UrlOption, PartialOptionsGroup, OptionsGroup, PartialOptionsTree, UtxoCohortObject, AddrCohortObject, CohortObject, CohortGroupObject, FetchedLineSeriesBlueprint, FetchedBaselineSeriesBlueprint, FetchedHistogramSeriesBlueprint, FetchedDotsBaselineSeriesBlueprint, PatternAll, PatternFull, PatternWithAdjusted, PatternWithPercentiles, PatternBasic, PatternBasicWithMarketCap, PatternBasicWithoutMarketCap, PatternWithoutRelative, CohortAll, CohortFull, CohortWithAdjusted, CohortWithPercentiles, CohortBasic, CohortBasicWithMarketCap, CohortBasicWithoutMarketCap, CohortWithoutRelative, CohortAddr, CohortLongTerm, CohortAgeRange, CohortAgeRangeWithMatured, CohortGroupFull, CohortGroupWithAdjusted, CohortGroupWithPercentiles, CohortGroupLongTerm, CohortGroupAgeRange, CohortGroupBasic, CohortGroupBasicWithMarketCap, CohortGroupBasicWithoutMarketCap, CohortGroupWithoutRelative, CohortGroupAddr, UtxoCohortGroupObject, AddrCohortGroupObject, FetchedDotsSeriesBlueprint, PartialHeatmapOption, HeatmapOption, FetchedCandlestickSeriesBlueprint, FetchedPriceSeriesBlueprint, AnyPricePattern, AnyValuePattern } from "./options/partial.js"
  *
  *
  * @import { UnitObject as Unit } from "./utils/units.js"

website/src/heatmap/controls/dates.js

@@ -0,0 +1,170 @@
+import { createSelect } from "../../../scripts/utils/dom.js";
+import { GENESIS_DATE, todayISODate, toISODate } from "../time.js";
+import { createHeatmapPersistedValue, findChoiceByKey } from "./shared.js";
+
+/**
+ * @typedef {Object} RangeChoice
+ * @property {string} label
+ * @property {string} date
+ */
+
+/**
+ * @param {HeatmapOption} option
+ * @param {(range: { from: string, to: string }) => void} onChange
+ */
+export function createDateControls(option, onChange) {
+  const currentYear = new Date().getUTCFullYear();
+  const fromChoices = createFromChoices(currentYear);
+  const toChoices = createToChoices(currentYear);
+  const fallbackFromChoice = fromChoices.at(-1) ?? fromChoices[0];
+  const fallbackToChoice = toChoices[0];
+  const defaultFromChoice = findChoiceByKey(
+    fromChoices,
+    option.defaults?.from ?? "",
+    fallbackFromChoice,
+    rangeChoiceLabel,
+  );
+  const defaultToChoice = findChoiceByKey(
+    toChoices,
+    option.defaults?.to ?? "",
+    fallbackToChoice,
+    rangeChoiceLabel,
+  );
+
+  const persistedFrom = createHeatmapPersistedValue(
+    option,
+    "from",
+    "hm_from",
+    rangeChoiceLabel(defaultFromChoice),
+  );
+  const persistedTo = createHeatmapPersistedValue(
+    option,
+    "to",
+    "hm_to",
+    rangeChoiceLabel(defaultToChoice),
+  );
+
+  let fromChoice = findChoiceByKey(
+    fromChoices,
+    persistedFrom.value,
+    defaultFromChoice,
+    rangeChoiceLabel,
+  );
+  let toChoice = findChoiceByKey(
+    toChoices,
+    persistedTo.value,
+    defaultToChoice,
+    rangeChoiceLabel,
+  );
+
+  if (fromChoice.date > toChoice.date) {
+    toChoice = findSameLabelChoice(toChoices, fromChoice, defaultToChoice);
+  }
+  persistDateChoices();
+
+  const fromSelect = createSelect({
+    id: "heatmap-from",
+    label: "from",
+    choices: fromChoices,
+    initialValue: fromChoice,
+    onChange(choice) {
+      fromChoice = choice;
+      if (fromChoice.date > toChoice.date) {
+        toChoice = findSameLabelChoice(toChoices, fromChoice, defaultToChoice);
+        toSelect.set(toChoice);
+      }
+      persistDateChoices();
+      onChange({ from: fromChoice.date, to: toChoice.date });
+    },
+    toKey: rangeChoiceLabel,
+    toLabel: rangeChoiceLabel,
+  });
+  const toSelect = createSelect({
+    id: "heatmap-to",
+    label: "to",
+    choices: toChoices,
+    initialValue: toChoice,
+    onChange(choice) {
+      toChoice = choice;
+      if (fromChoice.date > toChoice.date) {
+        fromChoice = findSameLabelChoice(
+          fromChoices,
+          toChoice,
+          defaultFromChoice,
+        );
+        fromSelect.set(fromChoice);
+      }
+      persistDateChoices();
+      onChange({ from: fromChoice.date, to: toChoice.date });
+    },
+    toKey: rangeChoiceLabel,
+    toLabel: rangeChoiceLabel,
+  });
+
+  return {
+    elements: [fromSelect.element, toSelect.element],
+    from: fromChoice.date,
+    to: toChoice.date,
+  };
+
+  function persistDateChoices() {
+    persistedFrom.setImmediate(rangeChoiceLabel(fromChoice));
+    persistedTo.setImmediate(rangeChoiceLabel(toChoice));
+  }
+}
+
+/**
+ * @param {number} currentYear
+ * @returns {RangeChoice[]}
+ */
+function createFromChoices(currentYear) {
+  const choices = [{ label: "genesis", date: GENESIS_DATE }];
+  for (let year = 2009; year <= currentYear; year++) {
+    choices.push({
+      label: String(year),
+      date: year === 2009 ? GENESIS_DATE : yearStartISODate(year),
+    });
+  }
+  return choices;
+}
+
+/**
+ * @param {number} currentYear
+ * @returns {RangeChoice[]}
+ */
+function createToChoices(currentYear) {
+  const today = todayISODate();
+  const todayTime = Date.parse(`${today}T00:00:00Z`);
+  const choices = [{ label: "today", date: today }];
+  for (let year = currentYear; year >= 2009; year--) {
+    choices.push({ label: String(year), date: yearEndISODate(year, todayTime) });
+  }
+  return choices;
+}
+
+/** @param {RangeChoice} choice */
+function rangeChoiceLabel(choice) {
+  return choice.label;
+}
+
+/**
+ * @param {readonly RangeChoice[]} choices
+ * @param {RangeChoice} choice
+ * @param {RangeChoice} fallback
+ */
+function findSameLabelChoice(choices, choice, fallback) {
+  return choices.find((candidate) => candidate.label === choice.label) ?? fallback;
+}
+
+/** @param {number} year */
+function yearStartISODate(year) {
+  return toISODate(new Date(Date.UTC(year, 0, 1)));
+}
+
+/**
+ * @param {number} year
+ * @param {number} todayTime
+ */
+function yearEndISODate(year, todayTime) {
+  return toISODate(new Date(Math.min(Date.UTC(year, 11, 31), todayTime)));
+}

website/src/heatmap/controls/index.js

@@ -0,0 +1,38 @@
+import { createDateControls } from "./dates.js";
+import { createYControls } from "./y.js";
+
+/**
+ * @typedef {Object} HeatmapControlSelection
+ * @property {string} from
+ * @property {string} to
+ * @property {number | undefined} yMin
+ * @property {number | undefined} yMax
+ */
+
+/**
+ * @param {Object} args
+ * @param {(range: { from: string, to: string }) => void} args.onRangeChange
+ * @param {(range: { yMin: number | undefined, yMax: number | undefined }) => void} args.onYRangeChange
+ */
+export function createHeatmapControls({ onRangeChange, onYRangeChange }) {
+  const element = document.createElement("fieldset");
+
+  return {
+    element,
+    /**
+     * @param {HeatmapOption} option
+     * @returns {HeatmapControlSelection}
+     */
+    setOption(option) {
+      const dates = createDateControls(option, onRangeChange);
+      const y = createYControls(option, onYRangeChange);
+      element.replaceChildren(...dates.elements, ...y.elements);
+      return {
+        from: dates.from,
+        to: dates.to,
+        yMin: y.yMin,
+        yMax: y.yMax,
+      };
+    },
+  };
+}

website/src/heatmap/controls/shared.js

@@ -0,0 +1,33 @@
+import { createPersistedValue } from "../../../scripts/utils/persisted.js";
+
+/**
+ * @param {HeatmapOption} option
+ * @param {string} key
+ * @param {string} urlKey
+ * @param {string} defaultValue
+ */
+export function createHeatmapPersistedValue(option, key, urlKey, defaultValue) {
+  return createPersistedValue({
+    defaultValue,
+    storageKey: `${heatmapStoragePrefix(option)}-${key}`,
+    urlKey,
+    serialize: (value) => value,
+    deserialize: (value) => value,
+  });
+}
+
+/**
+ * @template T
+ * @param {readonly T[]} choices
+ * @param {string} key
+ * @param {T} fallback
+ * @param {(choice: T) => string} toKey
+ */
+export function findChoiceByKey(choices, key, fallback, toKey) {
+  return choices.find((candidate) => toKey(candidate) === key) ?? fallback;
+}
+
+/** @param {HeatmapOption} option */
+function heatmapStoragePrefix(option) {
+  return `heatmap-${option.path.join("-")}`;
+}

website/src/heatmap/controls/y.js

@@ -0,0 +1,114 @@
+import { createSelect } from "../../../scripts/utils/dom.js";
+import { createHeatmapPersistedValue, findChoiceByKey } from "./shared.js";
+
+/**
+ * @param {HeatmapOption} option
+ * @param {(range: { yMin: number | undefined, yMax: number | undefined }) => void} onChange
+ */
+export function createYControls(option, onChange) {
+  const y = option.axis?.y;
+  const choices = y?.choices;
+  if (!choices || choices.length < 2) {
+    return { elements: [], yMin: undefined, yMax: undefined };
+  }
+
+  const fallbackMinChoice = choices[0];
+  const fallbackMaxChoice = choices.at(-1) ?? choices[0];
+  const defaultMinChoice = findChoiceByKey(
+    choices,
+    String(option.defaults?.yMin ?? ""),
+    fallbackMinChoice,
+    axisChoiceKey,
+  );
+  const defaultMaxChoice = findChoiceByKey(
+    choices,
+    String(option.defaults?.yMax ?? ""),
+    fallbackMaxChoice,
+    axisChoiceKey,
+  );
+  const persistedMin = createHeatmapPersistedValue(
+    option,
+    "y-min",
+    "hm_y_min",
+    axisChoiceKey(defaultMinChoice),
+  );
+  const persistedMax = createHeatmapPersistedValue(
+    option,
+    "y-max",
+    "hm_y_max",
+    axisChoiceKey(defaultMaxChoice),
+  );
+
+  let minChoice = findChoiceByKey(
+    choices,
+    persistedMin.value,
+    defaultMinChoice,
+    axisChoiceKey,
+  );
+  let maxChoice = findChoiceByKey(
+    choices,
+    persistedMax.value,
+    defaultMaxChoice,
+    axisChoiceKey,
+  );
+  if (minChoice.value > maxChoice.value) {
+    maxChoice = minChoice;
+  }
+  persistYChoices();
+
+  const minSelect = createSelect({
+    id: "heatmap-y-min",
+    label: "min",
+    choices,
+    initialValue: minChoice,
+    onChange(choice) {
+      minChoice = choice;
+      if (minChoice.value > maxChoice.value) {
+        maxChoice = minChoice;
+        maxSelect.set(maxChoice);
+      }
+      persistYChoices();
+      onChange({ yMin: minChoice.value, yMax: maxChoice.value });
+    },
+    toKey: axisChoiceKey,
+    toLabel: axisChoiceLabel,
+  });
+  const maxSelect = createSelect({
+    id: "heatmap-y-max",
+    label: "max",
+    choices: Array.from(choices).reverse(),
+    initialValue: maxChoice,
+    onChange(choice) {
+      maxChoice = choice;
+      if (minChoice.value > maxChoice.value) {
+        minChoice = maxChoice;
+        minSelect.set(minChoice);
+      }
+      persistYChoices();
+      onChange({ yMin: minChoice.value, yMax: maxChoice.value });
+    },
+    toKey: axisChoiceKey,
+    toLabel: axisChoiceLabel,
+  });
+
+  return {
+    elements: [minSelect.element, maxSelect.element],
+    yMin: minChoice.value,
+    yMax: maxChoice.value,
+  };
+
+  function persistYChoices() {
+    persistedMin.setImmediate(axisChoiceKey(minChoice));
+    persistedMax.setImmediate(axisChoiceKey(maxChoice));
+  }
+}
+
+/** @param {HeatmapAxisChoice} choice */
+function axisChoiceKey(choice) {
+  return String(choice.value);
+}
+
+/** @param {HeatmapAxisChoice} choice */
+function axisChoiceLabel(choice) {
+  return choice.label;
+}

website/src/heatmap/demo.js

@@ -1,6 +1,3 @@
-/** @import { PartialHeatmapOption } from "../../scripts/options/types.js" */
-/** @import { HeatmapPoints } from "./types.js" */
-
 import { createAverageGrid } from "./grid.js";
 import { INFERNO_LUT, intensityColor } from "./lut.js";
 import { GENESIS_DATE, todayISODate } from "./time.js";

website/src/heatmap/grid.js

@@ -1,5 +1,3 @@
-/** @import { HeatmapGrid, HeatmapGridFactory, HeatmapRange } from "./types.js" */
-
 /**
  * Generic date/y binning with average merge semantics.
  *

website/src/heatmap/index.js

@@ -1,52 +1,32 @@
-/** @import { HeatmapOption } from "../../scripts/options/types.js" */
+import { createHeader } from "../../scripts/utils/dom.js";
-/** @import { HeatmapAxisChoice, HeatmapGrid, HeatmapPoints } from "./types.js" */
-
-import { createHeader, createSelect } from "../../scripts/utils/dom.js";
 import { heatmapElement } from "../../scripts/utils/elements.js";
-import { createPersistedValue } from "../../scripts/utils/persisted.js";
 import { debounce, next } from "../../scripts/utils/timing.js";
+import { createHeatmapControls } from "./controls/index.js";
+import { createHeatmapLoader } from "./loader.js";
 import { createRenderer } from "./renderer.js";
-import { dateRange, GENESIS_DATE, todayISODate, toISODate } from "./time.js";
 import { createTooltipView } from "./tooltip/view.js";
 
-/**
- * @typedef {Object} RangeChoice
- * @property {string} label
- * @property {string} date
- */
-
-const MAX_PARALLEL_FETCHES = 8;
-
 /** @type {ReturnType<typeof createRenderer> | undefined} */
 let renderer;
 /** @type {HTMLCanvasElement | undefined} */
 let canvas;
 /** @type {ReturnType<typeof createTooltipView> | undefined} */
 let tooltipView;
+/** @type {ReturnType<typeof createHeatmapControls> | undefined} */
+let controls;
+/** @type {ReturnType<typeof createHeatmapLoader> | undefined} */
+let loader;
 /** @type {HTMLHeadingElement | undefined} */
 let headingElement;
-/** @type {HTMLElement | undefined} */
-let rangeControlsElement;
-/** @type {HTMLElement[]} */
-let dateControlElements = [];
-/** @type {HTMLElement[]} */
-let yControlElements = [];
 /** @type {HeatmapOption | undefined} */
 let currentOption;
 /** @type {HeatmapGrid | undefined} */
 let currentGrid;
-/** @type {string[]} */
-let currentDates = [];
-/** @type {Map<string, HeatmapPoints>} */
-let pointsByDate = new Map();
-/** @type {AbortController | undefined} */
-let abortController;
 const dirtyCols = new Set();
-let loadGeneration = 0;
 let paintScheduled = false;
 let initialized = false;
-let from = yearStartISODate(new Date().getUTCFullYear());
+let from = "";
-let to = todayISODate();
+let to = "";
 /** @type {number | undefined} */
 let yMin;
 /** @type {number | undefined} */
@@ -62,12 +42,28 @@ export function init() {
   const header = createHeader();
   headingElement = header.headingElement;
   const { headerElement } = header;
+  controls = createHeatmapControls({
+    onRangeChange(range) {
+      from = range.from;
+      to = range.to;
+      hideTooltip();
+      loadRange();
+    },
+    onYRangeChange(range) {
+      yMin = range.yMin;
+      yMax = range.yMax;
+      hideTooltip();
+      rebuildGrid();
+    },
+  });
+
   heatmapElement.append(headerElement);
-  heatmapElement.append(createRangeControls());
+  heatmapElement.append(controls.element);
 
   canvas = document.createElement("canvas");
   heatmapElement.append(canvas);
   renderer = createRenderer(canvas);
+  loader = createHeatmapLoader({ addDateToGrid, rebuildGrid, paint });
   tooltipView = createTooltipView(heatmapElement);
 
   canvas.addEventListener("pointermove", updateHoverTooltip);
@@ -89,10 +85,14 @@ export function setOption(option) {
   init();
   if (currentOption !== option) {
     currentOption = option;
-    pointsByDate = new Map();
+    loader?.reset();
-    updateDateControls(option);
+    const selection = controls?.setOption(option);
-    updateYControls(option);
+    if (selection) {
-    renderRangeControls();
+      from = selection.from;
+      to = selection.to;
+      yMin = selection.yMin;
+      yMax = selection.yMax;
+    }
     if (headingElement) headingElement.textContent = option.title;
     hideTooltip();
   }
@@ -106,127 +106,34 @@ function resizeAndRebuild() {
 }
 
 function loadRange() {
-  if (!currentOption) return;
+  if (!currentOption || !loader) return;
-
+  loader.load({ option: currentOption, from, to });
-  abortController?.abort();
-  const generation = ++loadGeneration;
-  const option = currentOption;
-  const controller = new AbortController();
-  abortController = controller;
-  currentDates = dateRange(from, to);
-
-  /** @type {{ date: string, dateIndex: number }[]} */
-  const missing = [];
-  for (let dateIndex = 0; dateIndex < currentDates.length; dateIndex++) {
-    const date = currentDates[dateIndex];
-    if (!pointsByDate.has(date)) missing.push({ date, dateIndex });
-  }
-  if (!missing.length) {
-    rebuildGrid();
-    abortController = undefined;
-    return;
-  }
-
-  let cursor = 0;
-  let needsRebuild = false;
-  const workers = Array.from({
-    length: Math.min(MAX_PARALLEL_FETCHES, missing.length),
-  }).map(async () => {
-    let index = nextMissingIndex();
-    while (index !== undefined) {
-      const entry = missing[index];
-      try {
-        const points = await option.points.fetch(
-          entry.date,
-          controller.signal,
-          (points) => {
-            if (isCurrentLoad(option, controller, generation)) {
-              setPoints(entry, points);
-            }
-          },
-        );
-        if (isCurrentLoad(option, controller, generation)) {
-          setPoints(entry, points);
-        }
-      } catch (error) {
-        if (controller.signal.aborted) return;
-        console.error(`Failed to fetch heatmap points for ${entry.date}`, error);
-      }
-      index = nextMissingIndex();
-    }
-  });
-
-  rebuildGrid();
-
-  void Promise.all(workers).then(() => {
-    if (isCurrentLoad(option, controller, generation)) {
-      if (needsRebuild) {
-        rebuildGrid();
-      } else {
-        paint();
-      }
-    }
-  });
-
-  function nextMissingIndex() {
-    if (cursor >= missing.length) return undefined;
-    const index = cursor;
-    cursor += 1;
-    return index;
-  }
-
-  /**
-   * @param {{ date: string, dateIndex: number }} entry
-   * @param {HeatmapPoints} points
-   */
-  function setPoints(entry, points) {
-    const previous = pointsByDate.get(entry.date);
-    if (previous && samePoints(previous, points)) return;
-    pointsByDate.set(entry.date, points);
-    if (previous) {
-      needsRebuild = true;
-    } else {
-      addDateToGrid(entry.dateIndex, points);
-    }
-  }
-}
-
-/**
- * @param {HeatmapOption} option
- * @param {AbortController} controller
- * @param {number} generation
- */
-function isCurrentLoad(option, controller, generation) {
-  return (
-    currentOption === option &&
-    abortController === controller &&
-    loadGeneration === generation &&
-    !controller.signal.aborted
-  );
 }
 
 function rebuildGrid() {
+  const dates = loader?.dates;
   if (
     !currentOption ||
     !renderer ||
+    !loader ||
+    !dates?.length ||
     renderer.width < 1 ||
-    renderer.height < 1 ||
+    renderer.height < 1
-    !currentDates.length
   ) {
     currentGrid = undefined;
     return;
   }
 
   currentGrid = currentOption.grid.create({
-    dates: currentDates,
+    dates,
     width: renderer.width,
     height: renderer.height,
     yMin,
     yMax,
   });
 
-  for (let i = 0; i < currentDates.length; i++) {
+  for (let i = 0; i < dates.length; i++) {
-    const points = pointsByDate.get(currentDates[i]);
+    const points = loader.getPoint(dates[i]);
     if (points) currentGrid.add(i, points);
   }
 
@@ -248,20 +155,6 @@ function addDateToGrid(dateIndex, points) {
   }
 }
 
-/**
- * @param {HeatmapPoints} a
- * @param {HeatmapPoints} b
- */
-function samePoints(a, b) {
-  if (a === b) return true;
-  if (a.kind !== b.kind || a.values !== b.values) return false;
-  if (a.kind === "implicit" && b.kind === "implicit") {
-    return a.yStart === b.yStart && a.yStep === b.yStep;
-  }
-  if (a.kind === "explicit" && b.kind === "explicit") return a.y === b.y;
-  return false;
-}
-
 /** @param {number} col */
 function schedulePaint(col) {
   dirtyCols.add(col);
@@ -270,9 +163,8 @@ function schedulePaint(col) {
   requestAnimationFrame(() => {
     paintScheduled = false;
     if (!dirtyCols.size) return;
-    const cols = Array.from(dirtyCols);
+    paint(dirtyCols);
     dirtyCols.clear();
-    paint(cols);
   });
 }
 
@@ -350,337 +242,3 @@ function updateTooltip(event, placement) {
 function hideTooltip() {
   tooltipView?.hide();
 }
-
-function createRangeControls() {
-  const fieldset = document.createElement("fieldset");
-  rangeControlsElement = fieldset;
-  return fieldset;
-}
-
-/** @param {HeatmapOption} option */
-function updateDateControls(option) {
-  const currentYear = new Date().getUTCFullYear();
-  const fromChoices = createFromChoices(currentYear);
-  const toChoices = createToChoices(currentYear);
-  const fallbackFromChoice = fromChoices.at(-1) ?? fromChoices[0];
-  const fallbackToChoice = toChoices[0];
-  const defaultFromChoice = findChoiceByKey(
-    fromChoices,
-    option.defaults?.from ?? "",
-    fallbackFromChoice,
-    rangeChoiceLabel,
-  );
-  const defaultToChoice = findChoiceByKey(
-    toChoices,
-    option.defaults?.to ?? "",
-    fallbackToChoice,
-    rangeChoiceLabel,
-  );
-
-  const persistedFrom = createHeatmapPersistedValue(
-    option,
-    "from",
-    "hm_from",
-    rangeChoiceLabel(defaultFromChoice),
-  );
-  const persistedTo = createHeatmapPersistedValue(
-    option,
-    "to",
-    "hm_to",
-    rangeChoiceLabel(defaultToChoice),
-  );
-
-  let fromChoice = findChoiceByKey(
-    fromChoices,
-    persistedFrom.value,
-    defaultFromChoice,
-    rangeChoiceLabel,
-  );
-  let toChoice = findChoiceByKey(
-    toChoices,
-    persistedTo.value,
-    defaultToChoice,
-    rangeChoiceLabel,
-  );
-
-  if (fromChoice.date > toChoice.date) {
-    toChoice = findSameLabelChoice(toChoices, fromChoice, defaultToChoice);
-  }
-  from = fromChoice.date;
-  to = toChoice.date;
-  persistDateChoices();
-
-  const fromSelect = createSelect({
-    id: "heatmap-from",
-    label: "from",
-    choices: fromChoices,
-    initialValue: fromChoice,
-    onChange(choice) {
-      fromChoice = choice;
-      if (fromChoice.date > toChoice.date) {
-        toChoice = findSameLabelChoice(toChoices, fromChoice, defaultToChoice);
-        toSelect.set(toChoice);
-      }
-      persistDateChoices();
-      setRange(fromChoice.date, toChoice.date);
-    },
-    toKey: rangeChoiceLabel,
-    toLabel: rangeChoiceLabel,
-  });
-  const toSelect = createSelect({
-    id: "heatmap-to",
-    label: "to",
-    choices: toChoices,
-    initialValue: toChoice,
-    onChange(choice) {
-      toChoice = choice;
-      if (fromChoice.date > toChoice.date) {
-        fromChoice = findSameLabelChoice(fromChoices, toChoice, defaultFromChoice);
-        fromSelect.set(fromChoice);
-      }
-      persistDateChoices();
-      setRange(fromChoice.date, toChoice.date);
-    },
-    toKey: rangeChoiceLabel,
-    toLabel: rangeChoiceLabel,
-  });
-
-  dateControlElements = [fromSelect.element, toSelect.element];
-
-  function persistDateChoices() {
-    persistedFrom.setImmediate(rangeChoiceLabel(fromChoice));
-    persistedTo.setImmediate(rangeChoiceLabel(toChoice));
-  }
-}
-
-/** @param {HeatmapOption} option */
-function updateYControls(option) {
-  const y = option.axis?.y;
-  const choices = y?.choices;
-  if (!choices || choices.length < 2) {
-    yMin = undefined;
-    yMax = undefined;
-    yControlElements = [];
-    return;
-  }
-
-  const fallbackMinChoice = choices[0];
-  const fallbackMaxChoice = choices.at(-1) ?? choices[0];
-  const defaultMinChoice = findChoiceByKey(
-    choices,
-    String(option.defaults?.yMin ?? ""),
-    fallbackMinChoice,
-    axisChoiceKey,
-  );
-  const defaultMaxChoice = findChoiceByKey(
-    choices,
-    String(option.defaults?.yMax ?? ""),
-    fallbackMaxChoice,
-    axisChoiceKey,
-  );
-  const persistedMin = createHeatmapPersistedValue(
-    option,
-    "y-min",
-    "hm_y_min",
-    axisChoiceKey(defaultMinChoice),
-  );
-  const persistedMax = createHeatmapPersistedValue(
-    option,
-    "y-max",
-    "hm_y_max",
-    axisChoiceKey(defaultMaxChoice),
-  );
-
-  let minChoice = findChoiceByKey(
-    choices,
-    persistedMin.value,
-    defaultMinChoice,
-    axisChoiceKey,
-  );
-  let maxChoice = findChoiceByKey(
-    choices,
-    persistedMax.value,
-    defaultMaxChoice,
-    axisChoiceKey,
-  );
-  if (minChoice.value > maxChoice.value) {
-    maxChoice = minChoice;
-  }
-  yMin = minChoice.value;
-  yMax = maxChoice.value;
-  persistYChoices();
-
-  const minSelect = createSelect({
-    id: "heatmap-y-min",
-    label: "min",
-    choices,
-    initialValue: minChoice,
-    onChange(choice) {
-      minChoice = choice;
-      if (minChoice.value > maxChoice.value) {
-        maxChoice = minChoice;
-        maxSelect.set(maxChoice);
-      }
-      persistYChoices();
-      setYRange(minChoice.value, maxChoice.value);
-    },
-    toKey: axisChoiceKey,
-    toLabel: axisChoiceLabel,
-  });
-  const maxSelect = createSelect({
-    id: "heatmap-y-max",
-    label: "max",
-    choices: Array.from(choices).reverse(),
-    initialValue: maxChoice,
-    onChange(choice) {
-      maxChoice = choice;
-      if (minChoice.value > maxChoice.value) {
-        minChoice = maxChoice;
-        minSelect.set(minChoice);
-      }
-      persistYChoices();
-      setYRange(minChoice.value, maxChoice.value);
-    },
-    toKey: axisChoiceKey,
-    toLabel: axisChoiceLabel,
-  });
-
-  yControlElements = [minSelect.element, maxSelect.element];
-
-  function persistYChoices() {
-    persistedMin.setImmediate(axisChoiceKey(minChoice));
-    persistedMax.setImmediate(axisChoiceKey(maxChoice));
-  }
-}
-
-function renderRangeControls() {
-  if (!rangeControlsElement) return;
-  rangeControlsElement.replaceChildren(
-    ...dateControlElements,
-    ...yControlElements,
-  );
-}
-
-/**
- * @param {number} currentYear
- * @returns {RangeChoice[]}
- */
-function createFromChoices(currentYear) {
-  const choices = [{ label: "genesis", date: GENESIS_DATE }];
-  for (let year = 2009; year <= currentYear; year++) {
-    choices.push({
-      label: String(year),
-      date: year === 2009 ? GENESIS_DATE : yearStartISODate(year),
-    });
-  }
-  return choices;
-}
-
-/**
- * @param {number} currentYear
- * @returns {RangeChoice[]}
- */
-function createToChoices(currentYear) {
-  const choices = [{ label: "today", date: todayISODate() }];
-  for (let year = currentYear; year >= 2009; year--) {
-    choices.push({ label: String(year), date: yearEndISODate(year) });
-  }
-  return choices;
-}
-
-/**
- * @param {RangeChoice} choice
- */
-function rangeChoiceLabel(choice) {
-  return choice.label;
-}
-
-/**
- * @param {readonly RangeChoice[]} choices
- * @param {RangeChoice} choice
- * @param {RangeChoice} fallback
- */
-function findSameLabelChoice(choices, choice, fallback) {
-  return choices.find((candidate) => candidate.label === choice.label) ?? fallback;
-}
-
-/**
- * @param {string} nextFrom
- * @param {string} nextTo
- */
-function setRange(nextFrom, nextTo) {
-  from = nextFrom;
-  to = nextTo;
-  hideTooltip();
-  loadRange();
-}
-
-/**
- * @param {number} nextYMin
- * @param {number} nextYMax
- */
-function setYRange(nextYMin, nextYMax) {
-  yMin = nextYMin;
-  yMax = nextYMax;
-  hideTooltip();
-  rebuildGrid();
-}
-
-/** @param {HeatmapAxisChoice} choice */
-function axisChoiceKey(choice) {
-  return String(choice.value);
-}
-
-/** @param {HeatmapAxisChoice} choice */
-function axisChoiceLabel(choice) {
-  return choice.label;
-}
-
-/** @param {HeatmapOption} option */
-function heatmapStoragePrefix(option) {
-  return `heatmap-${option.path.join("-")}`;
-}
-
-/**
- * @param {HeatmapOption} option
- * @param {string} key
- * @param {string} urlKey
- * @param {string} defaultValue
- */
-function createHeatmapPersistedValue(option, key, urlKey, defaultValue) {
-  return createPersistedValue({
-    defaultValue,
-    storageKey: `${heatmapStoragePrefix(option)}-${key}`,
-    urlKey,
-    serialize: (value) => value,
-    deserialize: (value) => value,
-  });
-}
-
-/**
- * @template T
- * @param {readonly T[]} choices
- * @param {string} key
- * @param {T} fallback
- * @param {(choice: T) => string} toKey
- */
-function findChoiceByKey(choices, key, fallback, toKey) {
-  return choices.find((candidate) => toKey(candidate) === key) ?? fallback;
-}
-
-/** @param {number} year */
-function yearStartISODate(year) {
-  return toISODate(new Date(Date.UTC(year, 0, 1)));
-}
-
-/** @param {number} year */
-function yearEndISODate(year) {
-  return toISODate(
-    new Date(
-      Math.min(
-        Date.UTC(year, 11, 31),
-        Date.parse(`${todayISODate()}T00:00:00Z`),
-      ),
-    ),
-  );
-}

website/src/heatmap/loader.js

@@ -0,0 +1,155 @@
+import { dateRange } from "./time.js";
+
+const MAX_PARALLEL_FETCHES = 8;
+
+/**
+ * @param {Object} args
+ * @param {(dateIndex: number, points: HeatmapPoints) => void} args.addDateToGrid
+ * @param {() => void} args.rebuildGrid
+ * @param {() => void} args.paint
+ */
+export function createHeatmapLoader({ addDateToGrid, rebuildGrid, paint }) {
+  /** @type {string[]} */
+  let dates = [];
+  /** @type {Map<string, HeatmapPoints>} */
+  let pointsByDate = new Map();
+  /** @type {AbortController | undefined} */
+  let abortController;
+  /** @type {HeatmapOption | undefined} */
+  let activeOption;
+  let generation = 0;
+
+  return {
+    get dates() {
+      return dates;
+    },
+    /** @param {string} date */
+    getPoint(date) {
+      return pointsByDate.get(date);
+    },
+    reset() {
+      pointsByDate = new Map();
+    },
+    /**
+     * @param {Object} args
+     * @param {HeatmapOption} args.option
+     * @param {string} args.from
+     * @param {string} args.to
+     */
+    load({ option, from, to }) {
+      abortController?.abort();
+      const controller = new AbortController();
+      const currentGeneration = ++generation;
+      activeOption = option;
+      abortController = controller;
+      dates = dateRange(from, to);
+
+      /** @type {{ date: string, dateIndex: number }[]} */
+      const missing = [];
+      for (let dateIndex = 0; dateIndex < dates.length; dateIndex++) {
+        const date = dates[dateIndex];
+        if (!pointsByDate.has(date)) missing.push({ date, dateIndex });
+      }
+
+      if (!missing.length) {
+        rebuildGrid();
+        abortController = undefined;
+        return;
+      }
+
+      let cursor = 0;
+      let needsRebuild = false;
+      const workers = Array.from({
+        length: Math.min(MAX_PARALLEL_FETCHES, missing.length),
+      }).map(async () => {
+        let index = nextMissingIndex();
+        while (index !== undefined) {
+          const entry = missing[index];
+          try {
+            const points = await option.points.fetch(
+              entry.date,
+              controller.signal,
+              (points) => {
+                if (isCurrentLoad(option, controller, currentGeneration)) {
+                  setPoints(entry, points);
+                }
+              },
+            );
+            if (isCurrentLoad(option, controller, currentGeneration)) {
+              setPoints(entry, points);
+            }
+          } catch (error) {
+            if (controller.signal.aborted) return;
+            console.error(
+              `Failed to fetch heatmap points for ${entry.date}`,
+              error,
+            );
+          }
+          index = nextMissingIndex();
+        }
+      });
+
+      rebuildGrid();
+
+      void Promise.all(workers).then(() => {
+        if (isCurrentLoad(option, controller, currentGeneration)) {
+          if (needsRebuild) {
+            rebuildGrid();
+          } else {
+            paint();
+          }
+        }
+      });
+
+      function nextMissingIndex() {
+        if (cursor >= missing.length) return undefined;
+        const index = cursor;
+        cursor += 1;
+        return index;
+      }
+
+      /**
+       * @param {{ date: string, dateIndex: number }} entry
+       * @param {HeatmapPoints} points
+       */
+      function setPoints(entry, points) {
+        const previous = pointsByDate.get(entry.date);
+        if (previous && samePoints(previous, points)) return;
+        pointsByDate.set(entry.date, points);
+        if (previous) {
+          needsRebuild = true;
+        } else {
+          addDateToGrid(entry.dateIndex, points);
+        }
+      }
+    },
+  };
+
+  /**
+   * @param {HeatmapOption} option
+   * @param {AbortController} controller
+   * @param {number} currentGeneration
+   */
+  function isCurrentLoad(option, controller, currentGeneration) {
+    return (
+      activeOption === option &&
+      abortController === controller &&
+      generation === currentGeneration &&
+      !controller.signal.aborted
+    );
+  }
+}
+
+/**
+ * @param {HeatmapPoints} a
+ * @param {HeatmapPoints} b
+ */
+function samePoints(a, b) {
+  if (a === b) return true;
+  if (a.kind !== b.kind || a.values !== b.values) return false;
+  if (a.kind === "implicit" && b.kind === "implicit") {
+    return a.yStart === b.yStart && a.yStep === b.yStep;
+  }
+  if (a.kind === "explicit" && b.kind === "explicit") return a.y === b.y;
+  return false;
+}

website/src/heatmap/lut.js

@@ -1,5 +1,3 @@
-/** @import { HeatmapColorFn } from "./types.js" */
-
 const INFERNO_STOPS = [
   [0, 0, 0, 0],
   [0.13, 40, 11, 84],

website/src/heatmap/oracle.js

@@ -1,6 +1,3 @@
-/** @import { PartialHeatmapOption } from "../../scripts/options/types.js" */
-/** @import { HeatmapPoints } from "./types.js" */
-
 import { brk } from "../../scripts/utils/client.js";
 import { createAverageGrid } from "./grid.js";
 import { INFERNO_LUT, logIntensityColor } from "./lut.js";

website/src/heatmap/tooltip/index.js

@@ -1,5 +1,3 @@
-/** @import { HeatmapTooltipFn } from "../types.js" */
-
 import { numberToShortUSFormat } from "../../../scripts/utils/format.js";
 
 /**