git: reset

app/src/scripts/datasets/consts/address.ts

@@ -0,0 +1,30 @@
+export const addressCohortsBySize = [
+  {
+    key: "plankton",
+    name: "Plankton",
+  },
+  {
+    key: "shrimp",
+    name: "Shrimp",
+  },
+  { key: "crab", name: "Crab" },
+  { key: "fish", name: "Fish" },
+  { key: "shark", name: "Shark" },
+  { key: "whale", name: "Whale" },
+  { key: "humpback", name: "Humpback" },
+  { key: "megalodon", name: "Megalodon" },
+] as const;
+
+export const addressCohortsByType = [
+  { key: "p2pk", name: "P2PK" },
+  { key: "p2pkh", name: "P2PKH" },
+  { key: "p2sh", name: "P2SH" },
+  { key: "p2wpkh", name: "P2WPKH" },
+  { key: "p2wsh", name: "P2WSH" },
+  { key: "p2tr", name: "P2TR" },
+] as const;
+
+export const addressCohorts = [
+  ...addressCohortsBySize,
+  ...addressCohortsByType,
+] as const;

app/src/scripts/datasets/consts/age.ts

@@ -0,0 +1,147 @@
+export const xthCohorts = [
+  {
+    key: "sth",
+    name: "Short Term Holders",
+    legend: "STH",
+  },
+  {
+    key: "lth",
+    name: "Long Term Holders",
+    legend: "LTH",
+  },
+] as const;
+
+export const upToCohorts = [
+  { key: "up_to_1d", name: "Up To 1 Day", legend: "1D" },
+  { key: "up_to_1w", name: "Up To 1 Week", legend: "1W" },
+  { key: "up_to_1m", name: "Up To 1 Month", legend: "1M" },
+  { key: "up_to_2m", name: "Up To 2 Months", legend: "2M" },
+  { key: "up_to_3m", name: "Up To 3 Months", legend: "3M" },
+  { key: "up_to_4m", name: "Up To 4 Months", legend: "4M" },
+  { key: "up_to_5m", name: "Up To 5 Months", legend: "5M" },
+  { key: "up_to_6m", name: "Up To 6 Months", legend: "6M" },
+  { key: "up_to_1y", name: "Up To 1 Year", legend: "1Y" },
+  { key: "up_to_2y", name: "Up To 2 Years", legend: "2Y" },
+  { key: "up_to_3y", name: "Up To 3 Years", legend: "3Y" },
+  { key: "up_to_5y", name: "Up To 5 Years", legend: "5Y" },
+  { key: "up_to_7y", name: "Up To 7 Years", legend: "7Y" },
+  { key: "up_to_10y", name: "Up To 10 Years", legend: "10Y" },
+  { key: "up_to_15y", name: "Up To 15 Years", legend: "15Y" },
+] as const;
+
+export const fromXToYCohorts = [
+  {
+    key: "from_1d_to_1w",
+    name: "From 1 Day To 1 Week",
+    legend: "1D - 1W",
+  },
+  {
+    key: "from_1w_to_1m",
+    name: "From 1 Week To 1 Month",
+    legend: "1W - 1M",
+  },
+  {
+    key: "from_1m_to_3m",
+    name: "From 1 Month To 3 Months",
+    legend: "1M - 3M",
+  },
+  {
+    key: "from_3m_to_6m",
+    name: "From 3 Months To 6 Months",
+    legend: "3M - 6M",
+  },
+  {
+    key: "from_6m_to_1y",
+    name: "From 6 Months To 1 Year",
+    legend: "6M - 1Y",
+  },
+  {
+    key: "from_1y_to_2y",
+    name: "From 1 Year To 2 Years",
+    legend: "1Y - 2Y",
+  },
+  {
+    key: "from_2y_to_3y",
+    name: "From 2 Years To 3 Years",
+    legend: "2Y - 3Y",
+  },
+  {
+    key: "from_3y_to_5y",
+    name: "From 3 Years To 5 Years",
+    legend: "3Y - 5Y",
+  },
+  {
+    key: "from_5y_to_7y",
+    name: "From 5 Years To 7 Years",
+    legend: "5Y - 7Y",
+  },
+  {
+    key: "from_7y_to_10y",
+    name: "From 7 Years To 10 Years",
+    legend: "7Y - 10Y",
+  },
+  {
+    key: "from_10y_to_15y",
+    name: "From 10 Years To 15 Years",
+    legend: "10Y - 15Y",
+  },
+] as const;
+
+export const fromXCohorts = [
+  {
+    key: "from_1y",
+    name: "From 1 Year",
+    legend: "1Y+",
+  },
+  {
+    key: "from_2y",
+    name: "From 2 Years",
+    legend: "2Y+",
+  },
+  {
+    key: "from_4y",
+    name: "From 4 Years",
+    legend: "4Y+",
+  },
+  {
+    key: "from_10y",
+    name: "From 10 Years",
+    legend: "10Y+",
+  },
+  {
+    key: "from_15y",
+    name: "From 15 Years",
+    legend: "15Y+",
+  },
+] as const;
+
+export const yearCohorts = [
+  { key: "year_2009", name: "2009" },
+  { key: "year_2010", name: "2010" },
+  { key: "year_2011", name: "2011" },
+  { key: "year_2012", name: "2012" },
+  { key: "year_2013", name: "2013" },
+  { key: "year_2014", name: "2014" },
+  { key: "year_2015", name: "2015" },
+  { key: "year_2016", name: "2016" },
+  { key: "year_2017", name: "2017" },
+  { key: "year_2018", name: "2018" },
+  { key: "year_2019", name: "2019" },
+  { key: "year_2020", name: "2020" },
+  { key: "year_2021", name: "2021" },
+  { key: "year_2022", name: "2022" },
+  { key: "year_2023", name: "2023" },
+  { key: "year_2024", name: "2024" },
+] as const;
+
+export const ageCohorts = [
+  {
+    key: "",
+    name: "",
+  },
+  ...xthCohorts,
+  ...upToCohorts,
+  ...fromXToYCohorts,
+  ...fromXCohorts,
+  ...yearCohorts,
+] as const;

app/src/scripts/datasets/consts/averages.ts

@@ -0,0 +1,15 @@
+export const averages = [
+  { name: "1 Week", key: "1w", days: 7 },
+  { name: "8 Days", key: "8d", days: 8 },
+  { name: "13 Days", key: "13d", days: 13 },
+  { name: "21 Days", key: "21d", days: 21 },
+  { name: "1 Month", key: "1m", days: 30 },
+  { name: "34 Days", key: "34d", days: 34 },
+  { name: "55 Days", key: "55d", days: 55 },
+  { name: "89 Days", key: "89d", days: 89 },
+  { name: "144 Days", key: "144d", days: 144 },
+  { name: "1 Year", key: "1y", days: 365 },
+  { name: "2 Years", key: "2y", days: 2 * 365 },
+  { name: "200 Weeks", key: "200w", days: 200 * 7 },
+  { name: "4 Years", key: "4y", days: 4 * 365 },
+] as const;

app/src/scripts/datasets/consts/liquidities.ts

@@ -0,0 +1,11 @@
+export const liquidities = [
+  {
+    key: "illiquid",
+    name: "Illiquid",
+  },
+  { key: "liquid", name: "Liquid" },
+  {
+    key: "highly_liquid",
+    name: "Highly Liquid",
+  },
+] as const;

app/src/scripts/datasets/consts/percentiles.ts

@@ -0,0 +1,116 @@
+export const percentiles = [
+  {
+    key: "median_price_paid",
+    name: "Median",
+    title: "Median Paid",
+    value: 50,
+  },
+  {
+    key: "95p_price_paid",
+    name: `95%`,
+    title: `95th Percentile Paid`,
+    value: 95,
+  },
+  {
+    key: "90p_price_paid",
+    name: `90%`,
+    title: `90th Percentile Paid`,
+    value: 90,
+  },
+  {
+    key: "85p_price_paid",
+    name: `85%`,
+    title: `85th Percentile Paid`,
+    value: 85,
+  },
+  {
+    key: "80p_price_paid",
+    name: `80%`,
+    title: `80th Percentile Paid`,
+    value: 80,
+  },
+  {
+    key: "75p_price_paid",
+    name: `75%`,
+    title: `75th Percentile Paid`,
+    value: 75,
+  },
+  {
+    key: "70p_price_paid",
+    name: `70%`,
+    title: `70th Percentile Paid`,
+    value: 70,
+  },
+  {
+    key: "65p_price_paid",
+    name: `65%`,
+    title: `65th Percentile Paid`,
+    value: 65,
+  },
+  {
+    key: "60p_price_paid",
+    name: `60%`,
+    title: `60th Percentile Paid`,
+    value: 60,
+  },
+  {
+    key: "55p_price_paid",
+    name: `55%`,
+    title: `55th Percentile Paid`,
+    value: 55,
+  },
+  {
+    key: "45p_price_paid",
+    name: `45%`,
+    title: `45th Percentile Paid`,
+    value: 45,
+  },
+  {
+    key: "40p_price_paid",
+    name: `40%`,
+    title: `40th Percentile Paid`,
+    value: 40,
+  },
+  {
+    key: "35p_price_paid",
+    name: `35%`,
+    title: `35th Percentile Paid`,
+    value: 35,
+  },
+  {
+    key: "30p_price_paid",
+    name: `30%`,
+    title: `30th Percentile Paid`,
+    value: 30,
+  },
+  {
+    key: "25p_price_paid",
+    name: `25%`,
+    title: `25th Percentile Paid`,
+    value: 25,
+  },
+  {
+    key: "20p_price_paid",
+    name: `20%`,
+    title: `20th Percentile Paid`,
+    value: 20,
+  },
+  {
+    key: "15p_price_paid",
+    name: `15%`,
+    title: `15th Percentile Paid`,
+    value: 15,
+  },
+  {
+    key: "10p_price_paid",
+    name: `10%`,
+    title: `10th Percentile Paid`,
+    value: 10,
+  },
+  {
+    key: "05p_price_paid",
+    name: `5%`,
+    title: `5th Percentile Paid`,
+    value: 5,
+  },
+] as const;

app/src/scripts/datasets/consts/returns.ts

@@ -0,0 +1,14 @@
+export const totalReturns = [
+  { name: "1 Day", key: "1d" },
+  { name: "1 Month", key: "1m" },
+  { name: "6 Months", key: "6m" },
+  { name: "1 Year", key: "1y" },
+  { name: "2 Years", key: "2y" },
+  { name: "3 Years", key: "3y" },
+  { name: "4 Years", key: "4y" },
+  { name: "6 Years", key: "6y" },
+  { name: "8 Years", key: "8y" },
+  { name: "10 Years", key: "10y" },
+] as const;
+
+export const compoundReturns = [{ name: "4 Years", key: "4y" }] as const;

app/src/scripts/datasets/consts/types.d.ts

@@ -0,0 +1,19 @@
+type AgeCohortKey = (typeof import("./age").ageCohorts)[number]["key"];
+
+type AddressCohortKey =
+  (typeof import("./address").addressCohorts)[number]["key"];
+
+type LiquidityKey = (typeof import("./liquidities").liquidities)[number]["key"];
+
+type AddressCohortKeySplitByLiquidity = `${LiquidityKey}_${AddressCohortKey}`;
+
+type AnyCohortKey = AgeCohortKey | AddressCohortKey;
+
+type AnyPossibleCohortKey = AnyCohortKey | AddressCohortKeySplitByLiquidity;
+
+type AverageName = (typeof import("./averages").averages)[number]["key"];
+
+type TotalReturnKey = (typeof import("./returns").totalReturns)[number]["key"];
+
+type CompoundReturnKey =
+  (typeof import("./returns").compoundReturns)[number]["key"];

app/src/scripts/datasets/date.ts

@@ -0,0 +1,41 @@
+import groupedKeysToPath from "/src/../../datasets/grouped_keys_to_url_path.json";
+
+import { createResourceDataset } from "./resource";
+
+export { averages } from "./consts/averages";
+
+export function createDateDatasets({
+  setActiveResources,
+}: {
+  setActiveResources: Setter<Set<ResourceDataset<any, any>>>;
+}) {
+  type Key = keyof typeof groupedKeysToPath.date;
+  type ResourceData = ReturnType<typeof createResourceDataset<"date">>;
+
+  const resourceDatasets = {} as Record<Exclude<Key, "ohlc">, ResourceData>;
+
+  Object.entries(groupedKeysToPath.date).forEach(([_key, path]) => {
+    const key = _key as Key;
+
+    if (key !== "ohlc") {
+      resourceDatasets[key] = createResourceDataset<"date">({
+        scale: "date",
+        path,
+        setActiveResources,
+      });
+    }
+  });
+
+  const price = createResourceDataset<"date", OHLC>({
+    scale: "date",
+    path: "/date-to-ohlc",
+    setActiveResources,
+  });
+
+  const datasets = {
+    price,
+    ...resourceDatasets,
+  };
+
+  return datasets;
+}

app/src/scripts/datasets/height.ts

@@ -0,0 +1,36 @@
+import groupedKeysToPath from "/src/../../datasets/grouped_keys_to_url_path.json";
+
+import { createResourceDataset } from "./resource";
+
+export function createHeightDatasets({
+  setActiveResources,
+}: {
+  setActiveResources: Setter<Set<ResourceDataset<any, any>>>;
+}) {
+  type Key = keyof typeof groupedKeysToPath.height;
+  type ResourceData = ReturnType<typeof createResourceDataset<"height">>;
+
+  const resourceDatasets = {} as Record<Exclude<Key, "ohlc">, ResourceData>;
+
+  Object.keys(groupedKeysToPath.height).forEach(([_key, path]) => {
+    const key = _key as Key;
+    if (key !== "ohlc") {
+      resourceDatasets[key] = createResourceDataset<"height">({
+        scale: "height",
+        path,
+        setActiveResources,
+      });
+    }
+  });
+
+  const price = createResourceDataset<"height", OHLC>({
+    scale: "height",
+    path: "/height-to-ohlc",
+    setActiveResources,
+  });
+
+  return {
+    ...resourceDatasets,
+    price,
+  };
+}

app/src/scripts/datasets/index.ts

@@ -0,0 +1,17 @@
+import { createDateDatasets } from "./date";
+import { createHeightDatasets } from "./height";
+
+export const scales = ["date" as const, "height" as const];
+
+export const HEIGHT_CHUNK_SIZE = 10_000;
+
+export function createDatasets({
+  setActiveResources,
+}: {
+  setActiveResources: Setter<Set<ResourceDataset<any, any>>>;
+}) {
+  return {
+    date: createDateDatasets({ setActiveResources }),
+    height: createHeightDatasets({ setActiveResources }),
+  } satisfies Record<ResourceScale, any>;
+}

app/src/scripts/datasets/resource.ts

@@ -0,0 +1,246 @@
+import { createLazyMemo } from "@solid-primitives/memo";
+
+import {
+  ONE_DAY_IN_MS,
+  ONE_HOUR_IN_MS,
+  ONE_MINUTE_IN_MS,
+} from "/src/scripts/utils/time";
+import { createRWS } from "/src/solid/rws";
+
+import { HEIGHT_CHUNK_SIZE } from ".";
+
+export function createResourceDataset<
+  Scale extends ResourceScale,
+  Type extends OHLC | number = number,
+>({
+  scale,
+  path,
+  setActiveResources,
+}: {
+  scale: Scale;
+  path: string;
+  setActiveResources: Setter<Set<ResourceDataset<any, any>>>;
+}) {
+  const baseURL = `${
+    location.hostname === "localhost"
+      ? "http://localhost:3110"
+      : "https://api.satonomics.xyz"
+  }${path}`;
+
+  type Dataset = Scale extends "date"
+    ? FetchedDateDataset<Type>
+    : FetchedHeightDataset<Type>;
+
+  type Value = DatasetValue<
+    Type extends number ? SingleValueData : CandlestickData
+  >;
+
+  const fetchedJSONs = new Array(
+    (new Date().getFullYear() - new Date("2009-01-01").getFullYear()) *
+      (scale === "date" ? 2 : 8),
+  )
+    .fill(null)
+    .map((): FetchedResult<Scale, Type> => {
+      const json = createRWS<FetchedJSON<Scale, Type, Dataset> | null>(null);
+
+      return {
+        at: null,
+        json,
+        loading: false,
+        vec: createMemo(() => {
+          const map = json()?.dataset.map || null;
+
+          const chunkId = json()?.chunk.id!;
+
+          if (!map) {
+            return null;
+          }
+
+          if (Array.isArray(map)) {
+            return map.map(
+              (value, index) =>
+                ({
+                  number: chunkId + index,
+                  time: (chunkId + index) as Time,
+                  ...(typeof value !== "number" && value !== null
+                    ? { ...(value as OHLC), value: value.close }
+                    : { value: value === null ? NaN : (value as number) }),
+                }) as any as Value,
+            );
+          } else {
+            return Object.entries(map).map(
+              ([date, value]) =>
+                ({
+                  number: new Date(date).valueOf() / ONE_DAY_IN_MS,
+                  time: date,
+                  ...(typeof value !== "number" && value !== null
+                    ? { ...(value as OHLC), value: value.close }
+                    : { value: value === null ? NaN : (value as number) }),
+                }) as any as Value,
+            );
+          }
+        }),
+      };
+    }) as FetchedResult<Scale, Type>[];
+
+  const _fetch = async (id: number) => {
+    const index =
+      scale === "date" ? id - 2009 : Math.floor(id / HEIGHT_CHUNK_SIZE);
+
+    if (
+      index < 0 ||
+      (scale === "date" && id > new Date().getUTCFullYear()) ||
+      (scale === "height" &&
+        id > 165 * 365 * (new Date().getUTCFullYear() - 2009))
+    ) {
+      return;
+    }
+
+    const fetched = fetchedJSONs.at(index);
+
+    if (!fetched || fetched.loading) {
+      return;
+    } else if (fetched.at) {
+      const diff = new Date().valueOf() - fetched.at.valueOf();
+
+      if (
+        diff < ONE_MINUTE_IN_MS ||
+        (index < fetchedJSONs.findLastIndex((json) => json.at) &&
+          diff < ONE_HOUR_IN_MS)
+      ) {
+        return;
+      }
+    }
+
+    fetched.loading = true;
+
+    let cache: Cache | undefined;
+
+    const urlWithQuery = `${baseURL}?chunk=${id}`;
+
+    if (!fetched.json()) {
+      try {
+        cache = await caches.open("resources");
+
+        const cachedResponse = await cache.match(urlWithQuery);
+
+        if (cachedResponse) {
+          const json = await convertResponseToJSON<Scale, Type>(cachedResponse);
+
+          if (json) {
+            console.log(`cache: ${path}?chunk=${id}`);
+
+            fetched.json.set(() => json);
+          }
+        }
+      } catch {}
+    }
+
+    try {
+      const fetchedResponse = await fetch(urlWithQuery);
+
+      if (!fetchedResponse.ok) {
+        fetched.loading = false;
+        return;
+      }
+
+      const clonedResponse = fetchedResponse.clone();
+
+      const json = await convertResponseToJSON<Scale, Type>(fetchedResponse);
+
+      if (json) {
+        console.log(`fetch: ${path}?chunk=${id}`);
+
+        const previousMap = fetched.json()?.dataset.map;
+        const newMap = json.dataset.map;
+
+        const previousLength = Object.keys(previousMap || []).length;
+        const newLength = Object.keys(newMap).length;
+
+        if (!newLength) {
+          fetched.loading = false;
+          return;
+        }
+
+        if (previousLength && previousLength <= newLength) {
+          const previousLastValue = Object.values(previousMap || []).at(-1);
+          const newLastValue = Object.values(newMap).at(-1);
+
+          if (typeof newLastValue === "number") {
+            if (previousLastValue === newLastValue) {
+              fetched.at = new Date();
+              fetched.loading = false;
+              return;
+            }
+          } else {
+            const previousLastOHLC = previousLastValue as OHLC;
+            const newLastOHLC = newLastValue as OHLC;
+
+            if (
+              previousLastOHLC.open === newLastOHLC.open &&
+              previousLastOHLC.high === newLastOHLC.high &&
+              previousLastOHLC.low === newLastOHLC.low &&
+              previousLastOHLC.close === newLastOHLC.close
+            ) {
+              fetched.loading = false;
+              fetched.at = new Date();
+              return;
+            }
+          }
+        }
+
+        fetched.json.set(() => json);
+
+        if (cache) {
+          cache.put(urlWithQuery, clonedResponse);
+        }
+      }
+    } catch {
+      fetched.loading = false;
+      return;
+    }
+
+    fetched.at = new Date();
+    fetched.loading = false;
+  };
+
+  const resource: ResourceDataset<Scale, Type> = {
+    scale,
+    url: baseURL,
+    fetch: _fetch,
+    fetchedJSONs,
+    values: createLazyMemo(() => {
+      setActiveResources((resources) => resources.add(resource));
+
+      onCleanup(() =>
+        setActiveResources((resources) => {
+          resources.delete(resource);
+          return resources;
+        }),
+      );
+
+      const flat = fetchedJSONs.flatMap((fetched) => fetched.vec() || []);
+
+      return flat;
+    }),
+    drop() {
+      fetchedJSONs.forEach((fetched) => {
+        fetched.at = null;
+        fetched.json.set(null);
+      });
+    },
+  };
+
+  return resource;
+}
+
+async function convertResponseToJSON<
+  Scale extends ResourceScale,
+  Type extends number | OHLC,
+>(response: Response) {
+  try {
+    return (await response.json()) as FetchedJSON<Scale, Type>;
+  } catch (_) {
+    return null;
+  }
+}

app/src/scripts/datasets/types.d.ts

@@ -0,0 +1,98 @@
+type Datasets = ReturnType<typeof import("./index").createDatasets>;
+
+type DateDatasets = Datasets["date"];
+type HeightDatasets = Datasets["height"];
+type AnyDatasets = DateDatasets | HeightDatasets;
+
+type ResourceScale = (typeof import("./index").scales)[index];
+
+type DatasetValue<T> = T & Numbered & Valued;
+
+interface Dataset<
+  Scale extends ResourceScale,
+  Value extends SingleValueData | CandlestickData = SingleValueData,
+> {
+  scale: Scale;
+  values: Accessor<DatasetValue<Value>[]>;
+}
+
+interface ResourceDataset<
+  Scale extends ResourceScale,
+  Type extends OHLC | number = number,
+  FetchedDataset extends
+    | FetchedDateDataset<Type>
+    | FetchedHeightDataset<Type> = Scale extends "date"
+    ? FetchedDateDataset<Type>
+    : FetchedHeightDataset<Type>,
+  Value extends SingleValueData | CandlestickData = Type extends number
+    ? SingleValueData
+    : CandlestickData,
+> extends Dataset<Scale, Value> {
+  url: string;
+  fetch: (id: number) => void;
+  fetchedJSONs: FetchedResult<Scale, Type>[];
+  drop: VoidFunction;
+}
+
+interface FetchedResult<
+  Scale extends ResourceScale,
+  Type extends number | OHLC,
+  Dataset extends
+    | FetchedDateDataset<Type>
+    | FetchedHeightDataset<Type> = Scale extends "date"
+    ? FetchedDateDataset<Type>
+    : FetchedHeightDataset<Type>,
+  Value extends DatasetValue<SingleValueData | CandlestickData> = DatasetValue<
+    Type extends number ? SingleValueData : CandlestickData
+  >,
+> {
+  at: Date | null;
+  json: RWS<FetchedJSON<Scale, Type, Dataset> | null>;
+  vec: Accessor<Value[] | null>;
+  loading: boolean;
+}
+
+interface FetchedJSON<
+  Scale extends ResourceScale,
+  Type extends number | OHLC,
+  Dataset extends
+    | FetchedDateDataset<Type>
+    | FetchedHeightDataset<Type> = Scale extends "date"
+    ? FetchedDateDataset<Type>
+    : FetchedHeightDataset<Type>,
+> {
+  source: FetchedSource;
+  chunk: FetchedChunk;
+  dataset: FetchedDataset<Scale, Type, Dataset>;
+}
+
+type FetchedSource = string;
+
+interface FetchedChunk {
+  id: number;
+  previous: string | null;
+  next: string | null;
+}
+
+interface FetchedDataset<
+  Scale extends ResourceScale,
+  Type extends number | OHLC,
+  Dataset extends
+    | FetchedDateDataset<Type>
+    | FetchedHeightDataset<Type> = Scale extends "date"
+    ? FetchedDateDataset<Type>
+    : FetchedHeightDataset<Type>,
+> {
+  version: number;
+  map: Dataset;
+}
+
+type FetchedDateDataset<T> = Record<string, T>;
+type FetchedHeightDataset<T> = T[];
+
+interface OHLC {
+  open: number;
+  high: number;
+  low: number;
+  close: number;
+}