Fix web-docs.element.dev deployment (#32922)

* Fix docs

* Switch to vitepress for doc generation

* Run doc build in CI

* Switch docs build to layered
This commit is contained in:
Michael Telatynski
2026-03-25 18:10:06 +01:00
committed by GitHub
parent ec47986ef5
commit b90a32bea4
33 changed files with 2635 additions and 2183 deletions
+7 -55
View File
@@ -16,77 +16,29 @@ jobs:
name: GitHub Pages name: GitHub Pages
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
steps: steps:
- name: Fetch element-web - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with: with:
path: element-web
persist-credentials: false
- name: Fetch matrix-js-sdk
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
repository: matrix-org/matrix-js-sdk
path: matrix-js-sdk
persist-credentials: false persist-credentials: false
- uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4
with: with:
package_json_file: element-web/package.json package_json_file: package.json
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6
with: with:
cache: "pnpm" cache: "pnpm"
cache-dependency-path: element-web/pnpm-lock.yaml cache-dependency-path: pnpm-lock.yaml
node-version: "lts/*" node-version: "lts/*"
- name: Generate automations docs - name: Fetch layered build
working-directory: element-web run: ./scripts/layered.sh
run: |
pnpm install --frozen-lockfile
pnpm node ./scripts/gen-workflow-mermaid.ts ../element-web ../matrix-js-sdk > docs/automations.md
echo "- [Automations](automations.md)" >> docs/SUMMARY.md
- name: Setup mdBook
uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2
with:
mdbook-version: "0.5.1"
- name: Install mdbook extensions
run: cargo install mdbook-combiner mdbook-mermaid
- name: Prepare docs
run: |
mkdir docs
mv element-web/README.md element-web/docs/
mv element-web/docs/lib docs/
mv element-web/docs "docs/Element Web"
mv matrix-js-sdk/README.md matrix-js-sdk/docs/
mv matrix-js-sdk/docs "docs/Matrix JS SDK"
sed -i -e 's/\.\.\/README.md/README.md/' docs/**/SUMMARY.md
mdbook-combiner -m docs
sed -i -E 's/^\t# (.+)$/- [\1]()/gm;t' SUMMARY.md
sed -i -E 's/^- \[(.+)]\(<>\)$/---\n# \1/gm;t' SUMMARY.md
sed -i -E 's/\t- \[Introduction]/- [Introduction]/gm;t' SUMMARY.md
cat <<EOF > docs/SUMMARY.md
# Summary
- [Introduction](<Element Web/README.md>)
EOF
cat SUMMARY.md >> docs/SUMMARY.md
mv element-web/book.toml .
- name: Build docs - name: Build docs
run: mdbook build run: pnpm run docs:build
- name: Upload artifact - name: Upload artifact
uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4
with: with:
path: ./book path: ./docs/.vitepress/dist
deploy: deploy:
environment: environment:
+3
View File
@@ -45,6 +45,9 @@ jobs:
- name: Rethemendex Check - name: Rethemendex Check
command: "rethemendex" command: "rethemendex"
assert-diff: true assert-diff: true
- name: Docs
install: layered
command: "docs:build"
name: ${{ matrix.name }} name: ${{ matrix.name }}
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
steps: steps:
+4 -1
View File
@@ -16,7 +16,6 @@ package-lock.json
.env .env
.env.* .env.*
coverage coverage
/book
/index.html /index.html
# version file and tarball created by `npm pack` / `yarn pack` # version file and tarball created by `npm pack` / `yarn pack`
/git-revision.txt /git-revision.txt
@@ -38,3 +37,7 @@ storybook-static
.nx/workspace-data .nx/workspace-data
.pnpm-store .pnpm-store
# vitepress
/docs/.vitepress/dist
/docs/.vitepress/cache
-19
View File
@@ -1,19 +0,0 @@
# Summary
- [Introduction](../README.md)
# Build/Debug
- [Native Node modules](native-node-modules.md)
- [Windows requirements](windows-requirements.md)
- [Debugging](debugging.md)
- [Using gdb](gdb.md)
# Distribution
- [Updates](updates.md)
- [Packaging](packaging.md)
# Setup
- [Config](config.md)
-15
View File
@@ -1,15 +0,0 @@
# Configuration
All Element Web options documented [here](https://github.com/vector-im/element-web/blob/develop/docs/config.md) can be used as well as the following:
---
The app contains a configuration file specified at build time using [these instructions](https://github.com/element-hq/element-web/blob/develop/apps/desktop/README.md#config).
This config can be overwritten by the end using by creating a `config.json` file at the paths described [here](https://github.com/element-hq/element-web/blob/develop/apps/desktop/README.md#user-specified-configjson).
After changing the config, the app will need to be exited fully (including via the task tray) and re-started.
---
1. `update_base_url`: Specifies the URL of the update server, see [document](https://github.com/element-hq/element-web/blob/develop/apps/desktop/docs/updates.md).
2. `web_base_url`: Specifies the Element Web URL when performing actions such as popout widget. Defaults to `https://app.element.io/`.
-32
View File
@@ -1,32 +0,0 @@
# Documentation for possible options in this file is at
# https://rust-lang.github.io/mdBook/format/config.html
[book]
title = "Element Web & Desktop"
authors = ["New Vector Ltd.", "The Matrix.org Foundation C.I.C."]
language = "en"
# The directory that documentation files are stored in
src = "docs"
[build]
# Prevent markdown pages from being automatically generated when they're
# linked to in SUMMARY.md
create-missing = false
[output.html]
# Remove the numbers that appear before each item in the sidebar, as they can
# get quite messy as we nest deeper
no-section-label = true
additional-css = ["docs/lib/custom.css"]
# The source code URL of the repository
git-repository-url = "https://github.com/element-hq/element-web"
# The path that the docs are hosted on
site-url = "/element-web/"
additional-js = ["docs/lib/mermaid.min.js", "docs/lib/mermaid-init.js"]
[preprocessor]
[preprocessor.mermaid]
command = "mdbook-mermaid"
+169
View File
@@ -0,0 +1,169 @@
import { withMermaid } from "vitepress-plugin-mermaid";
function customPathResolver(href: string, currentPath: string) {
const [link, fragment] = href.split("#", 2);
if (currentPath === "index.md") {
if (link.startsWith("./docs/")) {
return `../docs/${href.slice(7)}`;
} else if (link.startsWith("docs/")) {
return `../${href}`;
}
}
switch (link) {
case "../packages/shared-components/README.md":
return `../../docs/readme-shared-components.md#${fragment}`;
case "../apps/web/README.md":
return `../../docs/readme-element-web.md#${fragment}`;
case "../README.md":
return `../../docs/index.md#${fragment}`;
default:
return `https://github.com/element-hq/element-web/blob/develop/${href.split("/").pop()}`;
}
}
// https://vitepress.dev/reference/site-config
export default withMermaid({
title: "Element Web & Desktop docs",
description: "Documentation",
srcExclude: ["changelogs", "SUMMARY.md"],
rewrites: {
":file": "docs/:file",
"README": "index",
},
markdown: {
config: (md) => {
// Custom rule to fix links
const defaultRender =
md.renderer.rules.link_open ||
function (tokens, idx, options, env, self) {
return self.renderToken(tokens, idx, options);
};
md.renderer.rules.link_open = (tokens, idx, options, env, self) => {
const token = tokens[idx];
const hrefIndex = token.attrIndex("href");
if (hrefIndex >= 0) {
const href = token.attrs![hrefIndex][1];
if (!href.includes("://") && href.split("#", 2)[0].endsWith(".md")) {
token.attrs![hrefIndex][1] = customPathResolver(href, env.relativePath);
}
}
return defaultRender(tokens, idx, options, env, self);
};
},
},
themeConfig: {
nav: [
{ text: "Home", link: "/" },
{ text: "Website", link: "https://element.io/en" },
],
search: {
provider: "local",
},
sidebar: [
{
text: "README",
items: [
{ text: "Introduction", link: "/index" },
{ text: "Element Web", link: "/readme-element-web" },
{ text: "Element Desktop", link: "/readme-element-desktop" },
{ text: "Shared Components", link: "/readme-shared-components" },
],
},
{
text: "Usage",
items: [
{ text: "Betas", link: "/betas" },
{ text: "Labs", link: "/labs" },
],
},
{
text: "Setup",
items: [
{ text: "Install", link: "/install" },
{ text: "Config", link: "/config" },
{ text: "Custom home page", link: "/custom-home" },
{ text: "Kubernetes", link: "/kubernetes" },
{ text: "Jitsi", link: "/jitsi" },
{ text: "Encryption", link: "/e2ee" },
],
},
{
text: "Build",
items: [
{
text: "Web",
items: [
{ text: "Customisations", link: "/customisations" },
{ text: "Deprecated Modules", link: "/deprecated-modules" },
],
},
{
text: "Desktop",
items: [
{ text: "Native Node modules", link: "/native-node-modules" },
{ text: "Windows requirements", link: "/windows-requirements" },
{ text: "Debugging", link: "/debugging" },
{ text: "Using gdb", link: "/gdb" },
],
},
],
},
{
text: "Distribution",
items: [
{ text: "Updates", link: "/updates" },
{ text: "Packaging", link: "/packaging" },
],
},
{
text: "Contribution",
items: [
{ text: "Choosing an issue", link: "/choosing-an-issue" },
{ text: "Translation", link: "/translating" },
{ text: "Netlify builds", link: "/pr-previews" },
{ text: "Code review", link: "/review" },
],
},
{
text: "Development",
items: [
{ text: "App load order", link: "/app-load.md" },
{ text: "Translation", link: "/translating-dev.md" },
{ text: "Theming", link: "/theming.md" },
{ text: "Playwright end to end tests", link: "/playwright.md" },
{ text: "Memory profiling", link: "/memory-profiles-and-leaks.md" },
{ text: "Jitsi", link: "/jitsi-dev.md" },
{ text: "Feature flags", link: "/feature-flags.md" },
{ text: "OIDC and delegated authentication", link: "/oidc.md" },
{ text: "Release Process", link: "/release.md" },
{ text: "MVVM", link: "/MVVM.md" },
{ text: "Settings", link: "/settings.md" },
],
},
{
text: "Deep dive",
items: [
{ text: "Skinning", link: "/skinning" },
{ text: "Cider editor", link: "/ciderEditor" },
{ text: "Iconography", link: "/icons" },
{ text: "Local echo", link: "/local-echo-dev" },
{ text: "Media", link: "/media-handling" },
{ text: "Room List Store", link: "/room-list-store" },
{ text: "Scrolling", link: "/scrolling" },
{ text: "Usercontent", link: "/usercontent" },
{ text: "Widget layouts", link: "/widget-layouts" },
{ text: "Automations", link: "/generated/automations" },
],
},
],
socialLinks: [{ icon: "github", link: "https://github.com/element-hq/element-web" }],
},
});
+1 -1
View File
@@ -1,4 +1,4 @@
# MVVM # MVVM v1
_Deprecated_, see [MVVM.md](./MVVM.md) for the current version. _Deprecated_, see [MVVM.md](./MVVM.md) for the current version.
+1 -1
View File
@@ -1,4 +1,4 @@
# MVVM # MVVM v2
General description of the pattern can be found [here](https://en.wikipedia.org/wiki/Model%E2%80%93view%E2%80%93viewmodel). But the gist of it is that you divide your code into three sections: General description of the pattern can be found [here](https://en.wikipedia.org/wiki/Model%E2%80%93view%E2%80%93viewmodel). But the gist of it is that you divide your code into three sections:
-56
View File
@@ -1,56 +0,0 @@
# Summary
- [Introduction](../README.md)
# Usage
- [Betas](betas.md)
- [Labs](labs.md)
# Setup
- [Install](install.md)
- [Config](config.md)
- [Custom home page](custom-home.md)
- [Kubernetes](kubernetes.md)
- [Jitsi](jitsi.md)
- [Encryption](e2ee.md)
# Build
- [Customisations](customisations.md)
- [Deprecated Modules](deprecated-modules.md)
- [Native Node modules](native-node-modules.md)
# Contribution
- [Choosing an issue](choosing-an-issue.md)
- [Translation](translating.md)
- [Netlify builds](pr-previews.md)
- [Code review](review.md)
# Development
- [App load order](app-load.md)
- [Translation](translating-dev.md)
- [Theming](theming.md)
- [Playwright end to end tests](playwright.md)
- [Memory profiling](memory-profiles-and-leaks.md)
- [Jitsi](jitsi-dev.md)
- [Feature flags](feature-flags.md)
- [OIDC and delegated authentication](oidc.md)
- [Release Process](release.md)
- [MVVM](MVVM.md)
- [Settings](settings.md)
# Deep dive
- [Skinning](skinning.md)
- [Cider editor](ciderEditor.md)
- [Iconography](icons.md)
- [Local echo](local-echo-dev.md)
- [Media](media-handling.md)
- [Room List Store](room-list-store.md)
- [Scrolling](scrolling.md)
- [Usercontent](usercontent.md)
- [Widget layouts](widget-layouts.md)
+1 -1
View File
@@ -3,7 +3,7 @@
So you want to contribute to Element Web? That is awesome! So you want to contribute to Element Web? That is awesome!
If you're not sure where to start, make sure you read If you're not sure where to start, make sure you read
[CONTRIBUTING.md](../CONTRIBUTING.md), and the [CONTRIBUTING.md](https://github.com/element-hq/element-web/blob/develop/CONTRIBUTING.md), and the
[Development](../README.md#development) and [Development](../README.md#development) and
[Setting up a dev environment](../README.md#setting-up-a-dev-environment) [Setting up a dev environment](../README.md#setting-up-a-dev-environment)
sections of the README. sections of the README.
+12
View File
@@ -605,3 +605,15 @@ The following are undocumented or intended for developer use only.
2. `sync_timeline_limit` 2. `sync_timeline_limit`
3. `dangerously_allow_unsafe_and_insecure_passwords` 3. `dangerously_allow_unsafe_and_insecure_passwords`
4. `latex_maths_delims`: An optional setting to override the default delimiters used for maths parsing. See https://github.com/matrix-org/matrix-react-sdk/pull/5939 for details. Only used when `feature_latex_maths` is enabled. 4. `latex_maths_delims`: An optional setting to override the default delimiters used for maths parsing. See https://github.com/matrix-org/matrix-react-sdk/pull/5939 for details. Only used when `feature_latex_maths` is enabled.
## Additional config options for Element Desktop
1. `update_base_url`: Specifies the URL of the update server, see [document](https://github.com/element-hq/element-web/blob/develop/apps/desktop/docs/updates.md).
2. `web_base_url`: Specifies the Element Web URL when performing actions such as popout widget. Defaults to `https://app.element.io/`.
---
The app contains a configuration file specified at build time using [these instructions](https://github.com/element-hq/element-web/blob/develop/apps/desktop/README.md#config).
This config can be overwritten by the end using by creating a `config.json` file at the paths described [here](https://github.com/element-hq/element-web/blob/develop/apps/desktop/README.md#user-specified-configjson).
After changing the config, the app will need to be exited fully (including via the task tray) and re-started.
+1
View File
@@ -0,0 +1 @@
<!-- @content -->
+18
View File
@@ -0,0 +1,18 @@
import genWorkflowMermaid from "../../scripts/gen-workflow-mermaid";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const __dirname = dirname(fileURLToPath(import.meta.url));
export default {
async paths() {
const root = join(__dirname, "..", "..");
return [
{
params: { id: "automations" },
content: await genWorkflowMermaid([root, join(root, "node_modules", "matrix-js-sdk")]),
},
];
},
};
+1 -1
View File
@@ -24,7 +24,7 @@ const MyComponent = () => {
} }
``` ```
If possible, use the icon classes from [here](../res/css/compound/_Icon.pcss). If possible, use the icon classes from [here](https://github.com/element-hq/element-web/blob/develop/apps/web/res/css/compound/_Icon.pcss).
## Custom styling ## Custom styling
+1
View File
@@ -0,0 +1 @@
<!--@include: ../README.md-->
+2 -2
View File
@@ -1,6 +1,6 @@
# Installing Element Web # Installing Element Web
**Familiarise yourself with the [Important Security Notes](../README.md#important-security-notes) before starting, they apply to all installation methods.** **Familiarise yourself with the [Important Security Notes](../apps/web/README.md#important-security-notes) before starting, they apply to all installation methods.**
_Note: that for the security of your chats will need to serve Element over HTTPS. _Note: that for the security of your chats will need to serve Element over HTTPS.
Major browsers also do not allow you to use VoIP/video chats over HTTP, as WebRTC is only usable over HTTPS. Major browsers also do not allow you to use VoIP/video chats over HTTP, as WebRTC is only usable over HTTPS.
@@ -11,7 +11,7 @@ There are some exceptions like when using localhost, which is considered a [secu
1. Download the latest version from <https://github.com/element-hq/element-web/releases> 1. Download the latest version from <https://github.com/element-hq/element-web/releases>
1. Untar the tarball on your web server 1. Untar the tarball on your web server
1. Move (or symlink) the `element-x.x.x` directory to an appropriate name 1. Move (or symlink) the `element-x.x.x` directory to an appropriate name
1. Configure the correct caching headers in your webserver (see [README.md](../README.md#caching-requirements)) 1. Configure the correct caching headers in your webserver (see [README.md](../apps/web/README.md#caching-requirements))
1. Configure the app by copying `config.sample.json` to `config.json` and 1. Configure the app by copying `config.sample.json` to `config.json` and
modifying it. See the [configuration docs](config.md) for details. modifying it. See the [configuration docs](config.md) for details.
1. Enter the URL into your browser and log into Element! 1. Enter the URL into your browser and log into Element!
-14
View File
@@ -1,14 +0,0 @@
/* Prevent collapsible headings from wrapping onto two lines eagerly */
summary > h1,
summary > h2,
summary > h3,
summary > h4,
summary > h5,
summary > h6 {
display: inline-block;
}
/* Prevent longer checkbox lists from wrapping eagerly */
input + p {
display: inline;
}
-1
View File
@@ -1 +0,0 @@
mermaid.initialize({ startOnLoad:true });
-1648
View File
File diff suppressed because one or more lines are too long
-22
View File
@@ -1,27 +1,5 @@
# Playwright in Element Web # Playwright in Element Web
## Contents
- [Overview](#overview)
- [Prerequisites](#prerequisites)
- [Running the Tests](#running-the-tests)
- [Element Web E2E Tests](#element-web-e2e-tests)
- [Shared Components Tests](#shared-components-tests)
- [Projects](#projects)
- [How the Tests Work](#how-the-tests-work)
- [Test Structure](#test-structure)
- [Homeserver Setup](#homeserver-setup)
- [Fixtures](#fixtures)
- [Writing Tests](#writing-tests)
- [Getting a Homeserver](#getting-a-homeserver)
- [Logging In](#logging-in)
- [Joining a Room](#joining-a-room)
- [Using matrix-js-sdk](#using-matrix-js-sdk)
- [Best Practices](#best-practices)
- [Visual Testing](#visual-testing)
- [Test Tags](#test-tags)
- [Supported Container Runtimes](#supported-container-runtimes)
## Overview ## Overview
Element Web contains two sets of Playwright tests: Element Web contains two sets of Playwright tests:
+1
View File
@@ -0,0 +1 @@
<!--@include: ../apps/desktop/README.md-->
+1
View File
@@ -0,0 +1 @@
<!--@include: ../apps/web/README.md-->
+1
View File
@@ -0,0 +1 @@
<!--@include: ../packages/shared-components/README.md-->
+7 -1
View File
@@ -19,7 +19,10 @@
"lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) -print -exec action-validator {} ';'", "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) -print -exec action-validator {} ';'",
"lint:knip": "knip", "lint:knip": "knip",
"install:git-hooks": "husky", "install:git-hooks": "husky",
"postinstall": "node scripts/pnpm-link.ts && pnpm run -r sane-postinstall" "postinstall": "node scripts/pnpm-link.ts && pnpm run -r sane-postinstall",
"docs:dev": "vitepress dev docs",
"docs:build": "vitepress build docs",
"docs:preview": "vitepress preview docs"
}, },
"resolutions": { "resolutions": {
"pretty-format@30>react-is": "19.2.4", "pretty-format@30>react-is": "19.2.4",
@@ -51,10 +54,13 @@
"knip": "5.87.0", "knip": "5.87.0",
"lint-staged": "^16.0.0", "lint-staged": "^16.0.0",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"mermaid": "^11.13.0",
"minimist": "^1.2.6", "minimist": "^1.2.6",
"nx": "22.5.4", "nx": "22.5.4",
"prettier": "3.8.1", "prettier": "3.8.1",
"typescript": "catalog:", "typescript": "catalog:",
"vitepress": "^1.6.4",
"vitepress-plugin-mermaid": "^2.0.17",
"yaml": "^2.3.3" "yaml": "^2.3.3"
}, },
"pnpm": { "pnpm": {
+2173 -101
View File
File diff suppressed because it is too large Load Diff
+231 -212
View File
@@ -1,19 +1,10 @@
#!/usr/bin/env -S npx ts-node
import fs from "node:fs"; import fs from "node:fs";
import path from "node:path"; import path from "node:path";
import YAML from "yaml"; import YAML from "yaml";
import parseArgs from "minimist"; import parseArgs from "minimist";
import cronstrue from "cronstrue"; import cronstrue from "cronstrue";
import _ from "lodash"; import _ from "lodash";
import url from "node:url";
const argv = parseArgs<{
debug: boolean;
on: string | string[];
}>(process.argv.slice(2), {
string: ["on"],
boolean: ["debug"],
});
/** /**
* Generates unique ID strings (incremental base36) representing the given inputs. * Generates unique ID strings (incremental base36) representing the given inputs.
@@ -97,13 +88,13 @@ class Graph<T extends Node> {
// Removes nodes without any edges // Removes nodes without any edges
public cull(): void { public cull(): void {
const seenNodes = new Set<Node>(); const seenNodes = new Set<Node>();
graph.edges.forEach(([source, destination]) => { this.edges.forEach(([source, destination]) => {
seenNodes.add(source); seenNodes.add(source);
seenNodes.add(destination); seenNodes.add(destination);
}); });
graph.nodes.forEach((node) => { this.nodes.forEach((node) => {
if (!seenNodes.has(node)) { if (!seenNodes.has(node)) {
graph.nodes.delete(node.id); this.nodes.delete(node.id);
} }
}); });
} }
@@ -281,10 +272,10 @@ const triggers = new Map<string, Trigger>(); // keyed by trigger id
const projects = new Map<string, Project>(); // keyed by project name const projects = new Map<string, Project>(); // keyed by project name
const workflows = new Map<string, Workflow>(); // keyed by workflow name const workflows = new Map<string, Workflow>(); // keyed by workflow name
function getTriggerNodes<K extends keyof WorkflowYaml["on"]>(key: K, workflow: Workflow): Trigger[] { function getTriggerNodes<K extends keyof WorkflowYaml["on"]>(key: K, workflow: Workflow, on?: string[]): Trigger[] {
if (!TRIGGERS[key]) return []; if (!TRIGGERS[key]) return [];
if ((typeof argv.on === "string" || Array.isArray(argv.on)) && !toArray(argv.on).includes(key)) { if (on && !on.includes(key)) {
return []; return [];
} }
@@ -330,56 +321,6 @@ function shallowCompare(obj1: Record<string, any>, obj2: Record<string, any>): b
); );
} }
// Data ingest
for (const projectPath of argv._) {
const {
name,
repository: { url },
} = readJson<{ name: string; repository: { url: string } }>(projectPath, "package.json");
const workflowsPath = path.join(projectPath, ".github", "workflows");
const project: Project = {
name,
url,
path: projectPath,
workflows: new Map(),
};
for (const file of fs.readdirSync(workflowsPath).filter((f) => f.endsWith(".yml") || f.endsWith(".yaml"))) {
const data = readYaml<WorkflowYaml>(workflowsPath, file);
const name = data.name ?? file;
const workflow: Workflow = {
id: `${project.name}/${name}`,
name,
shape: "hexagon",
path: path.join(workflowsPath, file),
project,
link: `${project.url}/blob/develop/.github/workflows/${file}`,
on: data.on,
jobs: [],
};
for (const jobId in data.jobs) {
const job = data.jobs[jobId];
workflow.jobs.push({
id: `${workflow.name}/${jobId}`,
jobId,
name: job.name ?? jobId,
strategy: job.strategy,
needs: job.needs ? toArray(job.needs) : undefined,
shape: "subroutine",
link: `${project.url}/blob/develop/.github/workflows/${file}`,
});
}
project.workflows.set(name, workflow);
workflows.set(name, workflow);
}
projects.set(name, project);
}
class MermaidFlowchartPrinter { class MermaidFlowchartPrinter {
private static INDENT = 4; private static INDENT = 4;
private currentIndent = 0; private currentIndent = 0;
@@ -391,10 +332,11 @@ class MermaidFlowchartPrinter {
this.text += " ".repeat(this.currentIndent) + text + "\n"; this.text += " ".repeat(this.currentIndent) + text + "\n";
} }
public finish(): void { public finish(print: boolean): string {
this.indent(-1); this.indent(-1);
if (this.markdown) this.print("```\n"); if (this.markdown) this.print("```\n");
console.log(this.text); if (print) console.log(this.text);
return this.text;
} }
private indent(delta = 1): void { private indent(delta = 1): void {
@@ -485,151 +427,6 @@ class MermaidFlowchartPrinter {
} }
} }
const graph = new Graph<Workflow | Node>();
for (const workflow of workflows.values()) {
if (
(typeof argv.on === "string" || Array.isArray(argv.on)) &&
!toArray(argv.on).some((trigger) => trigger in workflow.on)
) {
continue;
}
graph.addNode(workflow);
Object.keys(workflow.on).forEach((trigger) => {
const nodes = getTriggerNodes(trigger as keyof WorkflowYaml["on"], workflow);
nodes.forEach((node) => {
graph.addNode(node);
graph.addEdge(node, workflow, "project" in node ? "workflow_run" : undefined);
});
});
}
// TODO separate disconnected nodes into their own graph
graph.cull();
// This is an awful hack to make the output graphs much better by allowing the splitting of certain nodes //
const bifurcatedNodes = [triggers.get("on:workflow_dispatch")].filter(Boolean) as Node[];
const removedEdgeMap = new Map<Node, Edge<any>[]>();
for (const node of bifurcatedNodes) {
removedEdgeMap.set(node, graph.removeNode(node));
}
const components = graph.components;
for (const node of bifurcatedNodes) {
const removedEdges = removedEdgeMap.get(node)!;
components.forEach((graph) => {
removedEdges.forEach((edge) => {
if (graph.nodes.has(edge[1].id)) {
graph.addNode(node);
graph.addEdge(...edge);
}
});
});
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////
if (argv.debug) {
debugGraph("global", graph);
}
components.forEach((graph) => {
const title = [...graph.roots]
.map((root) => root.name)
.join(" & ")
.replaceAll("<br>", " ");
const printer = new MermaidFlowchartPrinter("LR", title, true);
graph.nodes.forEach((node) => {
if ("project" in node) {
// TODO unsure about this edge
// if (node.jobs.length === 1) {
// printer.node(node);
// return;
// }
// TODO handle job.if on github.event_name
const subgraph = new Graph<Job>();
for (const job of node.jobs) {
subgraph.addNode(job);
if (job.needs) {
toArray(job.needs).forEach((req) => {
subgraph.addEdge(node.jobs.find((job) => job.jobId === req)!, job, "needs");
});
}
}
printer.subgraph(node.id, node.name, () => {
subgraph.edges.forEach(([source, destination, text]) => {
printer.edge(source, destination, text);
});
subgraph.nodes.forEach((job) => {
if (!job.strategy?.matrix) {
printer.node(job);
return;
}
let variations = cartesianProduct(
Object.keys(job.strategy.matrix)
.filter(
(key) =>
key !== "include" && key !== "exclude" && Array.isArray(job.strategy!.matrix[key]),
)
.map((matrixKey) => {
return job.strategy!.matrix[matrixKey].map((value) => ({ [matrixKey]: value }));
}),
)
.map((variation) => Object.assign({}, ...variation))
.filter((variation) => Object.keys(variation).length > 0);
if (job.strategy.matrix.include) {
variations.push(...job.strategy.matrix.include);
}
job.strategy.matrix.exclude?.forEach((exclusion) => {
variations = variations.filter((variation) => {
return !shallowCompare(exclusion, variation);
});
});
// TODO validate edge case
if (variations.length === 0) {
printer.node(job);
return;
}
const jobName = job.name.replace(/\${{.+}}/g, "").replace(/(?:\(\)| )+/g, " ");
printer.subgraph(job.id, jobName, () => {
variations.forEach((variation, i) => {
let variationName = job.name;
if (variationName.includes("${{ matrix.")) {
Object.keys(variation).map((key) => {
variationName = variationName.replace(`\${{ matrix.${key} }}`, variation[key]);
});
} else {
variationName = `${variationName} (${Object.values(variation).join(", ")})`;
}
printer.node({ ...job, id: `${job.id}-variation-${i}`, name: variationName });
});
});
});
});
return;
}
printer.node(node);
});
graph.edges.forEach(([sourceName, destinationName, text]) => {
printer.edge(sourceName, destinationName, text);
});
printer.finish();
if (argv.debug) {
printer.idGenerator.debug();
debugGraph("subgraph", graph);
}
});
function debugGraph(name: string, graph: Graph<any>): void { function debugGraph(name: string, graph: Graph<any>): void {
console.log("```"); console.log("```");
console.log(`## ${name}`); console.log(`## ${name}`);
@@ -638,3 +435,225 @@ function debugGraph(name: string, graph: Graph<any>): void {
console.log("```"); console.log("```");
console.log(""); console.log("");
} }
export default async function main(dirs: string[], on?: string[], print = false, debug = false): Promise<string> {
// Data ingest
for (const projectPath of dirs) {
const {
name,
repository: { url },
} = readJson<{ name: string; repository: { url: string } }>(projectPath, "package.json");
const workflowsPath = path.join(projectPath, ".github", "workflows");
const project: Project = {
name,
url,
path: projectPath,
workflows: new Map(),
};
for (const file of fs.readdirSync(workflowsPath).filter((f) => f.endsWith(".yml") || f.endsWith(".yaml"))) {
const data = readYaml<WorkflowYaml>(workflowsPath, file);
const name = data.name ?? file;
const workflow: Workflow = {
id: `${project.name}/${name}`,
name,
shape: "hexagon",
path: path.join(workflowsPath, file),
project,
link: `${project.url}/blob/develop/.github/workflows/${file}`,
on: data.on,
jobs: [],
};
for (const jobId in data.jobs) {
const job = data.jobs[jobId];
workflow.jobs.push({
id: `${workflow.name}/${jobId}`,
jobId,
name: job.name ?? jobId,
strategy: job.strategy,
needs: job.needs ? toArray(job.needs) : undefined,
shape: "subroutine",
link: `${project.url}/blob/develop/.github/workflows/${file}`,
});
}
project.workflows.set(name, workflow);
workflows.set(name, workflow);
}
projects.set(name, project);
}
const graph = new Graph<Workflow | Node>();
for (const workflow of workflows.values()) {
if (on && !on.some((trigger) => trigger in workflow.on)) {
continue;
}
graph.addNode(workflow);
Object.keys(workflow.on).forEach((trigger) => {
const nodes = getTriggerNodes(trigger as keyof WorkflowYaml["on"], workflow, on);
nodes.forEach((node) => {
graph.addNode(node);
graph.addEdge(node, workflow, "project" in node ? "workflow_run" : undefined);
});
});
}
// TODO separate disconnected nodes into their own graph
graph.cull();
// This is an awful hack to make the output graphs much better by allowing the splitting of certain nodes //
const bifurcatedNodes = [triggers.get("on:workflow_dispatch")].filter(Boolean) as Node[];
const removedEdgeMap = new Map<Node, Edge<any>[]>();
for (const node of bifurcatedNodes) {
removedEdgeMap.set(node, graph.removeNode(node));
}
const components = graph.components;
for (const node of bifurcatedNodes) {
const removedEdges = removedEdgeMap.get(node)!;
components.forEach((graph) => {
removedEdges.forEach((edge) => {
if (graph.nodes.has(edge[1].id)) {
graph.addNode(node);
graph.addEdge(...edge);
}
});
});
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////
if (debug) {
debugGraph("global", graph);
}
let text = "";
components.forEach((graph) => {
const title = [...graph.roots]
.map((root) => root.name)
.join(" & ")
.replaceAll("<br>", " ");
const printer = new MermaidFlowchartPrinter("LR", title, true);
graph.nodes.forEach((node) => {
if ("project" in node) {
// TODO unsure about this edge
// if (node.jobs.length === 1) {
// printer.node(node);
// return;
// }
// TODO handle job.if on github.event_name
const subgraph = new Graph<Job>();
for (const job of node.jobs) {
subgraph.addNode(job);
if (job.needs) {
toArray(job.needs).forEach((req) => {
subgraph.addEdge(node.jobs.find((job) => job.jobId === req)!, job, "needs");
});
}
}
printer.subgraph(node.id, node.name, () => {
subgraph.edges.forEach(([source, destination, text]) => {
printer.edge(source, destination, text);
});
subgraph.nodes.forEach((job) => {
if (!job.strategy?.matrix) {
printer.node(job);
return;
}
let variations = cartesianProduct(
Object.keys(job.strategy.matrix)
.filter(
(key) =>
key !== "include" &&
key !== "exclude" &&
Array.isArray(job.strategy!.matrix[key]),
)
.map((matrixKey) => {
return job.strategy!.matrix[matrixKey].map((value) => ({ [matrixKey]: value }));
}),
)
.map((variation) => Object.assign({}, ...variation))
.filter((variation) => Object.keys(variation).length > 0);
if (job.strategy.matrix.include) {
variations.push(...job.strategy.matrix.include);
}
job.strategy.matrix.exclude?.forEach((exclusion) => {
variations = variations.filter((variation) => {
return !shallowCompare(exclusion, variation);
});
});
// TODO validate edge case
if (variations.length === 0) {
printer.node(job);
return;
}
const jobName = job.name.replace(/\${{.+}}/g, "").replace(/(?:\(\)| )+/g, " ");
printer.subgraph(job.id, jobName, () => {
variations.forEach((variation, i) => {
let variationName = job.name;
if (variationName.includes("${{ matrix.")) {
Object.keys(variation).map((key) => {
variationName = variationName.replace(`\${{ matrix.${key} }}`, variation[key]);
});
} else {
variationName = `${variationName} (${Object.values(variation).join(", ")})`;
}
printer.node({ ...job, id: `${job.id}-variation-${i}`, name: variationName });
});
});
});
});
return;
}
printer.node(node);
});
graph.edges.forEach(([sourceName, destinationName, text]) => {
printer.edge(sourceName, destinationName, text);
});
text += printer.finish(print);
if (debug) {
printer.idGenerator.debug();
debugGraph("subgraph", graph);
}
});
return text;
}
if (import.meta.url.startsWith("file:")) {
const modulePath = url.fileURLToPath(import.meta.url);
if (process.argv[1] === modulePath) {
const argv = parseArgs<{
debug: boolean;
on: string | string[];
}>(process.argv.slice(2), {
string: ["on"],
boolean: ["debug"],
});
const on = typeof argv.on === "string" || Array.isArray(argv.on) ? toArray(argv.on) : undefined;
main(argv._, on, true, argv.debug)
.then((ret) => {
process.exit(0);
})
.catch((e) => {
console.error(e);
process.exit(1);
});
}
}