wip

bin/web/src/lib/analysis.spec.ts

@@ -0,0 +1,45 @@
+import { describe, it, expect } from 'vitest';
+import { EventType, parse_finished_report, Severity, type QualitativeWarning } from './analysis';
+import { parse_ndjson, type NewlineDeliminatedJson } from './ndjson';
+
+const SAMPLE_REPORT_NDJSON: NewlineDeliminatedJson = [
+    { "analyzers": [{ "name": "LTE SIB 6/7 Downgrade", "description": "Tests for LTE cells broadcasting a SIB type 6 and 7 which include 2G/3G frequencies with higher priorities." }, { "name": "IMSI Provided", "description": "Tests whether the UE's IMSI was ever provided to the cell" }, { "name": "Null Cipher", "description": "Tests whether the cell suggests using a null cipher (EEA0)" }, { "name": "Example Analyzer", "description": "Always returns true, if you are seeing this you are either a developer or you are about to have problems." }] },
+    { "timestamp": "2024-10-08T13:25:43.011689003-07:00", "skipped_message_reasons": ["DecodingError(UperDecodeError(Error { cause: BufferTooShort, msg: \"PerCodec:DecodeError:Requested Bits to decode 3, Remaining bits 1\", context: [] }))"], "analysis": [] },
+    { "timestamp": "2024-10-08T13:25:43.480872496-07:00", "skipped_message_reasons": [], "analysis": [{ "timestamp": "2024-08-19T03:33:54.318Z", "events": [null, null, null, { "event_type": { "type": "QualitativeWarning", "severity": "Low" }, "message": "TMSI was provided to cell" }] }] },
+];
+
+describe('analysis report parsing', () => {
+	it('parses the example analysis', () => {
+        const report = parse_finished_report(SAMPLE_REPORT_NDJSON);
+        expect(report.metadata.analyzers).toEqual([
+            {
+                "name":"LTE SIB 6/7 Downgrade",
+                "description":"Tests for LTE cells broadcasting a SIB type 6 and 7 which include 2G/3G frequencies with higher priorities.",
+            },
+            {
+                "name":"IMSI Provided",
+                "description":"Tests whether the UE's IMSI was ever provided to the cell",
+            },
+            {
+                "name":"Null Cipher",
+                "description":"Tests whether the cell suggests using a null cipher (EEA0)",
+            },
+            {
+                "name":"Example Analyzer",
+                "description":"Always returns true, if you are seeing this you are either a developer or you are about to have problems.",
+            }
+        ]);
+        expect(report.rows).toHaveLength(2);
+        expect(report.rows[0].skipped_message_reasons).toHaveLength(1);
+        expect(report.rows[0].analysis).toHaveLength(0);
+        expect(report.rows[1].skipped_message_reasons).toHaveLength(0);
+        expect(report.rows[1].analysis).toHaveLength(1);
+        expect(report.rows[1].analysis[0].events).toHaveLength(1);
+        const event = report.rows[1].analysis[0].events[0];
+        if (event.type === EventType.Warning) {
+            expect(event.severity).toEqual(Severity.Low);
+        } else {
+            throw 'wrong event type';
+        }
+	});
+});

bin/web/src/lib/analysis.ts

@@ -0,0 +1,99 @@
+import { parse_ndjson, type NewlineDeliminatedJson } from "./ndjson";
+import { req } from "./utils";
+
+export type AnalysisReport =
+    | LoadingReport
+    | FinishedReport;
+
+export type LoadingReport = {};
+
+export type FinishedReport = {
+    metadata: ReportMetadata;
+    rows: AnalysisRow[];
+};
+
+export type ReportMetadata = {
+    analyzers: AnalyzerMetadata[];
+};
+
+export type AnalyzerMetadata = {
+    name: string;
+    description: string;
+};
+
+export type AnalysisRow = {
+    timestamp: Date;
+    skipped_message_reasons: string[];
+    analysis: PacketAnalysis[];
+};
+
+export type PacketAnalysis = {
+    timestamp: Date;
+    events: Event[];
+};
+export type Event = QualitativeWarning | InformationalEvent;
+export enum EventType {
+    Informational,
+    Warning,
+}
+
+export type QualitativeWarning = {
+    type: EventType.Warning;
+    severity: Severity;
+    message: string;
+};
+
+export enum Severity {
+    Low,
+    Medium,
+    High,
+}
+
+export type InformationalEvent = {
+    type: EventType.Informational;
+    message: string;
+};
+
+export function parse_finished_report(report_json: NewlineDeliminatedJson): FinishedReport {
+    const metadata: ReportMetadata = report_json[0]; // this can be cast directly
+    const rows: AnalysisRow[] = report_json.slice(1).map((row_json: any) => {
+        const analysis: PacketAnalysis[] = row_json.analysis.map((analysis_json: any) => {
+            const events: Event[] = analysis_json.events.map((event_json: any): Event | null => {
+                    if (event_json === null) {
+                        return null;
+                    } else if (event_json.event_type === "Informational") {
+                        return {
+                            type: EventType.Informational,
+                            message: event_json.message,
+                        };
+                    } else {
+                        return {
+                            type: EventType.Warning,
+                            severity: event_json.severity === "High" ? Severity.High :
+                                event_json.severity === "Medium" ? Severity.Medium : Severity.Low,
+                            message: event_json.message,
+                        };
+                    }
+                })
+                .filter((maybe_event: Event | null) => maybe_event !== null);
+            return {
+                timestamp: analysis_json.timestamp,
+                events,
+            };
+        });
+        return {
+            timestamp: new Date(row_json.timestamp),
+            skipped_message_reasons: row_json.skipped_message_reasons,
+            analysis,
+        };
+    });
+    return {
+        metadata,
+        rows,
+    };
+}
+
+export async function get_report(name: string): Promise<FinishedReport> {
+    const report_json = parse_ndjson(await req('GET', `/api/analysis-report/${name}`));
+    return parse_finished_report(report_json);
+}

bin/web/src/lib/analysisManager.ts

@@ -0,0 +1,45 @@
+import type { Manifest, ManifestEntry } from "./manifest";
+import { req } from "./utils";
+
+export enum AnalysisStatus {
+    Running,
+    Queued,
+    Finished,
+}
+
+type AnalysisStatusJson = {
+    running: string | null;
+    queued: string[];
+    finished: string[];
+};
+
+export type AnalysisResult {
+    name: string,
+    status: AnalysisStatus,
+}
+
+export class AnalysisManager {
+    public analysis_status: Map<string, AnalysisStatus> = new Map();
+
+    public async run_analysis(name: string) {
+        await req('POST', `/api/analysis/${name}`);
+        this.analysis_status.set(name, AnalysisStatus.Queued);
+    }
+
+    public async update() {
+        this.analysis_status.clear();
+
+        const status: AnalysisStatusJson = JSON.parse(await req('GET', '/api/analysis'));
+        if (status.running) {
+            this.analysis_status.set(status.running, AnalysisStatus.Running);
+        }
+
+        for (const entry of status.queued) {
+            this.analysis_status.set(entry, AnalysisStatus.Queued);
+        }
+
+        for (const entry of status.finished) {
+            this.analysis_status.set(entry, AnalysisStatus.Finished);
+        }
+    }
+}

bin/web/src/lib/components/DownloadLink.svelte

@@ -0,0 +1,14 @@
+<script lang="ts">
+    let { url, text }: {
+        url: string;
+        text: string;
+    } = $props();
+</script>
+
+<a href={url}>📥 {text}</a>
+
+<style>
+    a {
+        @apply underline text-blue-400;
+    }
+</style>

bin/web/src/lib/components/ManifestTable.svelte

@@ -0,0 +1,40 @@
+<script lang="ts">
+    import { Manifest, ManifestEntry } from "$lib/manifest";
+    import TableRow from "./ManifestTableRow.svelte";
+    interface Props {
+        manifest: Manifest;
+    }
+    let { manifest }: Props = $props();
+</script>
+
+<table>
+    <thead>
+        <tr>
+            <th scope="col">Name</th>
+            <th scope="col">Date Started</th>
+            <th scope="col">Date of Last Message</th>
+            <th scope="col">Size (bytes)</th>
+            <th scope="col">PCAP</th>
+            <th scope="col">QMDL</th>
+            <th scope="col">Analysis Result</th>
+        </tr>
+    </thead>
+    <tbody>
+        {#if manifest.current_entry !== undefined}
+            <TableRow entry={manifest.current_entry} current={true} />
+        {/if}
+        {#each manifest.entries as entry}
+            <TableRow entry={entry} current={false} />
+        {/each}
+    </tbody>
+</table>
+
+<style>
+    table {
+        @apply table-auto border;
+    }
+
+    th {
+        @apply bg-gray-300 p-2;
+    }
+</style>

bin/web/src/lib/components/ManifestTableRow.svelte

@@ -0,0 +1,34 @@
+<script lang="ts">
+    import { ManifestEntry } from "$lib/manifest";
+    import DownloadLink from '$lib/components/DownloadLink.svelte';
+    let { entry, current }: {
+        entry: ManifestEntry;
+        current: boolean;
+    } = $props();
+
+    let row_class = current ? "current" : "";
+</script>
+
+<tr>
+    <th scope='row'>{entry.name}</th>
+    <td>{entry.start_time}</td>
+    <td>{entry.last_message_time}</td>
+    <td>{entry.qmdl_size_bytes}</td>
+    <td><DownloadLink url={entry.getPcapUrl()} text="pcap" /></td>
+    <td><DownloadLink url={entry.getQmdlUrl()} text="qmdl" /></td>
+    <td>N/A</td>
+</tr>
+
+<style>
+    th {
+        @apply font-bold p-2 border-b bg-blue-100;
+    }
+
+    td {
+        @apply p-2 border-b;
+    }
+
+    tr {
+        @apply even:bg-gray-100;
+    }
+</style>

bin/web/src/lib/manifest.ts

@@ -0,0 +1,59 @@
+interface JsonManifest {
+    entries: JsonManifestEntry[];
+    current_entry: JsonManifestEntry | null;
+}
+
+interface JsonManifestEntry {
+    name: string;
+    start_time: string;
+    last_message_time: string;
+    qmdl_size_bytes: number;
+    analysis_size_bytes: number;
+}
+
+export class Manifest {
+    public entries: ManifestEntry[] = [];
+    public current_entry: ManifestEntry | undefined;
+
+    constructor(json: JsonManifest) {
+        for (let entry of json.entries) {
+            this.entries.push(new ManifestEntry(entry));
+        }
+        if (json.current_entry !== null) {
+            this.current_entry = new ManifestEntry(json['current_entry']);
+        }
+
+        // sort entries in reverse chronological order
+        this.entries.reverse();
+    }
+}
+
+export class ManifestEntry {
+    public name: string;
+    public start_time: Date;
+    public last_message_time: Date | undefined;
+    public qmdl_size_bytes: number;
+    public analysis_size_bytes: number;
+
+    constructor(json: JsonManifestEntry) {
+        this.name = json.name;
+        this.qmdl_size_bytes = json.qmdl_size_bytes;
+        this.analysis_size_bytes = json.analysis_size_bytes;
+        this.start_time = new Date(json.start_time);
+        if (json.last_message_time !== undefined) {
+            this.last_message_time = new Date(json.last_message_time);
+        }
+    }
+
+    getPcapUrl(): string {
+        return `/api/pcap/${this.name}`;
+    }
+
+    getQmdlUrl(): string {
+        return `/api/qmdl/${this.name}`;
+    }
+
+    getAnalysisReportUrl(): string {
+        return `/api/analysis-report/${this.name}`;
+    }
+}

bin/web/src/lib/ndjson.spec.ts

@@ -0,0 +1,33 @@
+import { describe, it, expect } from 'vitest';
+import { parse_ndjson } from './ndjson';
+
+describe('parsing newline-deliminated json', () => {
+	it('parses normal JSON', () => {
+        const json = JSON.stringify({ foo: 100 });
+        const result = parse_ndjson(json);
+        expect(result).toHaveLength(1);
+        expect(result[0]).toEqual({ foo: 100 });
+	});
+
+	it('parses simple newline-deliminated json', () => {
+        const json_a = JSON.stringify({ a: 100 });
+        const json_b = JSON.stringify({ b: 200 });
+        const result = parse_ndjson(`${json_a}\n${json_b}`);
+        expect(result).toHaveLength(2);
+        expect(result[0]).toEqual({ a: 100 });
+        expect(result[1]).toEqual({ b: 200 });
+	})
+
+	it('parses newline-deliminated json with escaped newlines within', () => {
+    	const json_a = JSON.stringify({ a: 'this one has\n newlines and\nstuff' });
+        const json_b = JSON.stringify({ b: 200 });
+        const result = parse_ndjson(`${json_a}\n${json_b}`);
+        expect(result).toHaveLength(2);
+        expect(result[0]).toEqual({ a: 'this one has\n newlines and\nstuff' });
+        expect(result[1]).toEqual({ b: 200 });
+	})
+
+    it('actually errors out on invalid ndjson', () => {
+        expect(() =>  parse_ndjson("invalid\njson")).toThrow();
+    });
+});

bin/web/src/lib/ndjson.ts

@@ -0,0 +1,24 @@
+export type NewlineDeliminatedJson = any[];
+
+export function parse_ndjson(input: string): NewlineDeliminatedJson {
+    const lines = input.split('\n');
+    const result = [];
+    let current_line = '';
+    while (lines.length > 0) {
+        current_line += lines.shift();
+        try {
+            const entry = JSON.parse(current_line);
+            result.push(entry);
+            current_line = '';
+        } catch (e) {
+            // if this chunk wasn't valid JSON, assume there was an escaped
+            // newline in the JSON line, so simply continue to the next one.
+            // however, if we've reached the end of the input, that means we
+            // were given invalid nd-json
+            if (lines.length === 0) {
+                throw new Error(`unable to parse invalid nd-json: ${e}`);
+            }
+        }
+    }
+    return result;
+}

bin/web/src/lib/systemStats.ts

@@ -0,0 +1,19 @@
+export interface SystemStats {
+    disk_stats: DiskStats;
+    memory_stats: MemoryStats;
+}
+
+export interface DiskStats {
+    partition: string,
+    total_size: string,
+    used_size: string,
+    available_size: string,
+    used_percent: string,
+    mounted_on: string,
+}
+
+export interface MemoryStats {
+    total: string,
+    used: string,
+    free: string,
+}

bin/web/src/lib/utils.ts

@@ -0,0 +1,23 @@
+import { Manifest } from "./manifest";
+import type { SystemStats } from "./systemStats";
+
+export async function req(method: string, url: string): Promise<string> {
+    const response = await fetch(url, {
+        method: method,
+    });
+    const body = await response.text();
+    if (response.status >= 200 && response.status < 300) {
+        return body;
+    } else {
+        throw new Error(body);
+    }
+}
+
+export async function get_manifest(): Promise<Manifest> {
+    const manifest_json = JSON.parse(await req('GET', '/api/qmdl-manifest'));
+    return new Manifest(manifest_json);
+}
+
+export async function get_system_stats(): Promise<SystemStats> {
+    return JSON.parse(await req('GET', '/api/system-stats'));
+}