Improve Analytics with operational insights and temporal pattern panels

This commit is contained in:
Smittix
2026-02-19 12:59:39 +00:00
parent 16fa2b4270
commit 5af92efe47
5 changed files with 579 additions and 82 deletions
+184 -23
View File
@@ -1,23 +1,25 @@
"""Analytics dashboard: cross-mode summary, activity sparklines, export, geofence CRUD.""" """Analytics dashboard: cross-mode summary, activity sparklines, export, geofence CRUD."""
from __future__ import annotations from __future__ import annotations
import csv import csv
import io import io
import json import json
from datetime import datetime, timezone
from flask import Blueprint, Response, jsonify, request
from flask import Blueprint, Response, jsonify, request
import app as app_module
from utils.analytics import ( import app as app_module
from utils.analytics import (
get_activity_tracker, get_activity_tracker,
get_cross_mode_summary, get_cross_mode_summary,
get_emergency_squawks, get_emergency_squawks,
get_mode_health, get_mode_health,
) )
from utils.flight_correlator import get_flight_correlator from utils.alerts import get_alert_manager
from utils.geofence import get_geofence_manager from utils.flight_correlator import get_flight_correlator
from utils.temporal_patterns import get_pattern_detector from utils.geofence import get_geofence_manager
from utils.temporal_patterns import get_pattern_detector
analytics_bp = Blueprint('analytics', __name__, url_prefix='/analytics') analytics_bp = Blueprint('analytics', __name__, url_prefix='/analytics')
@@ -66,13 +68,172 @@ def analytics_squawks():
}) })
@analytics_bp.route('/patterns') @analytics_bp.route('/patterns')
def analytics_patterns(): def analytics_patterns():
"""Return detected temporal patterns.""" """Return detected temporal patterns."""
return jsonify({ return jsonify({
'status': 'success', 'status': 'success',
'patterns': get_pattern_detector().get_all_patterns(), 'patterns': get_pattern_detector().get_all_patterns(),
}) })
@analytics_bp.route('/insights')
def analytics_insights():
"""Return actionable insight cards and top changes."""
counts = get_cross_mode_summary()
tracker = get_activity_tracker()
sparklines = tracker.get_all_sparklines()
squawks = get_emergency_squawks()
patterns = get_pattern_detector().get_all_patterns()
alerts = get_alert_manager().list_events(limit=120)
top_changes = _compute_mode_changes(sparklines)
busiest_mode, busiest_count = _get_busiest_mode(counts)
critical_1h = _count_recent_alerts(alerts, severities={'critical', 'high'}, max_age_seconds=3600)
recurring_emitters = sum(1 for p in patterns if float(p.get('confidence') or 0.0) >= 0.7)
cards = []
if top_changes:
lead = top_changes[0]
direction = 'up' if lead['delta'] >= 0 else 'down'
cards.append({
'id': 'fastest_change',
'title': 'Fastest Change',
'value': f"{lead['mode_label']} ({lead['signed_delta']})",
'label': 'last window vs prior',
'severity': 'high' if lead['delta'] > 0 else 'low',
'detail': f"Traffic is trending {direction} in {lead['mode_label']}.",
})
else:
cards.append({
'id': 'fastest_change',
'title': 'Fastest Change',
'value': 'Insufficient data',
'label': 'wait for activity history',
'severity': 'low',
'detail': 'Sparklines need more samples to score momentum.',
})
cards.append({
'id': 'busiest_mode',
'title': 'Busiest Mode',
'value': f"{busiest_mode} ({busiest_count})",
'label': 'current observed entities',
'severity': 'medium' if busiest_count > 0 else 'low',
'detail': 'Highest live entity count across monitoring modes.',
})
cards.append({
'id': 'critical_alerts',
'title': 'Critical Alerts (1h)',
'value': str(critical_1h),
'label': 'critical/high severities',
'severity': 'critical' if critical_1h > 0 else 'low',
'detail': 'Prioritize triage if this count is non-zero.',
})
cards.append({
'id': 'emergency_squawks',
'title': 'Emergency Squawks',
'value': str(len(squawks)),
'label': 'active ADS-B emergency codes',
'severity': 'critical' if squawks else 'low',
'detail': 'Immediate aviation anomalies currently visible.',
})
cards.append({
'id': 'recurring_emitters',
'title': 'Recurring Emitters',
'value': str(recurring_emitters),
'label': 'pattern confidence >= 0.70',
'severity': 'medium' if recurring_emitters > 0 else 'low',
'detail': 'Potentially stationary or periodic emitters detected.',
})
return jsonify({
'status': 'success',
'generated_at': datetime.now(timezone.utc).isoformat(),
'cards': cards,
'top_changes': top_changes[:5],
})
def _compute_mode_changes(sparklines: dict[str, list[int]]) -> list[dict]:
mode_labels = {
'adsb': 'ADS-B',
'ais': 'AIS',
'wifi': 'WiFi',
'bluetooth': 'Bluetooth',
'dsc': 'DSC',
'acars': 'ACARS',
'vdl2': 'VDL2',
'aprs': 'APRS',
'meshtastic': 'Meshtastic',
}
rows = []
for mode, samples in (sparklines or {}).items():
if not isinstance(samples, list) or len(samples) < 4:
continue
window = max(2, min(12, len(samples) // 2))
recent = samples[-window:]
previous = samples[-(window * 2):-window]
if not previous:
continue
recent_avg = sum(recent) / len(recent)
prev_avg = sum(previous) / len(previous)
delta = round(recent_avg - prev_avg, 1)
rows.append({
'mode': mode,
'mode_label': mode_labels.get(mode, mode.upper()),
'delta': delta,
'signed_delta': ('+' if delta >= 0 else '') + str(delta),
'recent_avg': round(recent_avg, 1),
'previous_avg': round(prev_avg, 1),
'direction': 'up' if delta > 0 else ('down' if delta < 0 else 'flat'),
})
rows.sort(key=lambda r: abs(r['delta']), reverse=True)
return rows
def _count_recent_alerts(alerts: list[dict], severities: set[str], max_age_seconds: int) -> int:
now = datetime.now(timezone.utc)
count = 0
for event in alerts:
sev = str(event.get('severity') or '').lower()
if sev not in severities:
continue
created_raw = event.get('created_at')
if not created_raw:
continue
try:
created = datetime.fromisoformat(str(created_raw).replace('Z', '+00:00'))
except ValueError:
continue
if created.tzinfo is None:
created = created.replace(tzinfo=timezone.utc)
age = (now - created).total_seconds()
if 0 <= age <= max_age_seconds:
count += 1
return count
def _get_busiest_mode(counts: dict[str, int]) -> tuple[str, int]:
mode_labels = {
'adsb': 'ADS-B',
'ais': 'AIS',
'wifi': 'WiFi',
'bluetooth': 'Bluetooth',
'dsc': 'DSC',
'acars': 'ACARS',
'vdl2': 'VDL2',
'aprs': 'APRS',
'meshtastic': 'Meshtastic',
}
filtered = {k: int(v or 0) for k, v in (counts or {}).items() if k in mode_labels}
if not filtered:
return ('None', 0)
mode = max(filtered, key=filtered.get)
return (mode_labels.get(mode, mode.upper()), filtered[mode])
@analytics_bp.route('/export/<mode>') @analytics_bp.route('/export/<mode>')
+157 -11
View File
@@ -23,12 +23,112 @@
} }
} }
.analytics-grid { .analytics-grid {
display: grid; display: grid;
grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); grid-template-columns: repeat(auto-fill, minmax(140px, 1fr));
gap: var(--space-3, 12px); gap: var(--space-3, 12px);
margin-bottom: var(--space-4, 16px); margin-bottom: var(--space-4, 16px);
} }
.analytics-insight-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(210px, 1fr));
gap: var(--space-3, 12px);
}
.analytics-insight-card {
background: var(--bg-card, #151f2b);
border: 1px solid var(--border-color, #1e2d3d);
border-radius: var(--radius-md, 8px);
padding: 10px;
display: flex;
flex-direction: column;
gap: 4px;
}
.analytics-insight-card.low {
border-color: rgba(90, 106, 122, 0.5);
}
.analytics-insight-card.medium {
border-color: rgba(74, 163, 255, 0.45);
}
.analytics-insight-card.high {
border-color: rgba(214, 168, 94, 0.55);
}
.analytics-insight-card.critical {
border-color: rgba(226, 93, 93, 0.65);
}
.analytics-insight-card .insight-title {
font-size: 10px;
text-transform: uppercase;
letter-spacing: 0.04em;
color: var(--text-dim, #5a6a7a);
}
.analytics-insight-card .insight-value {
font-size: 16px;
font-weight: 700;
color: var(--text-primary, #e0e6ed);
line-height: 1.2;
}
.analytics-insight-card .insight-label {
font-size: 10px;
color: var(--text-secondary, #9aabba);
}
.analytics-insight-card .insight-detail {
font-size: 10px;
color: var(--text-dim, #5a6a7a);
}
.analytics-top-changes {
margin-top: 12px;
}
.analytics-change-row {
display: flex;
align-items: center;
gap: 10px;
padding: 7px 0;
border-bottom: 1px solid var(--border-color, #1e2d3d);
font-size: 10px;
}
.analytics-change-row:last-child {
border-bottom: none;
}
.analytics-change-row .mode {
min-width: 84px;
color: var(--text-primary, #e0e6ed);
font-weight: 600;
}
.analytics-change-row .delta {
min-width: 48px;
font-family: var(--font-mono, monospace);
}
.analytics-change-row .delta.up {
color: var(--accent-green, #38c180);
}
.analytics-change-row .delta.down {
color: var(--accent-red, #e25d5d);
}
.analytics-change-row .delta.flat {
color: var(--text-dim, #5a6a7a);
}
.analytics-change-row .avg {
color: var(--text-dim, #5a6a7a);
}
.analytics-card { .analytics-card {
background: var(--bg-card, #151f2b); background: var(--bg-card, #151f2b);
@@ -180,11 +280,57 @@
max-width: 60px; max-width: 60px;
} }
.analytics-correlation-pair .confidence-fill { .analytics-correlation-pair .confidence-fill {
height: 100%; height: 100%;
background: var(--accent-green, #38c180); background: var(--accent-green, #38c180);
border-radius: 2px; border-radius: 2px;
} }
.analytics-pattern-item {
padding: 8px;
border-bottom: 1px solid var(--border-color, #1e2d3d);
display: flex;
flex-direction: column;
gap: 4px;
}
.analytics-pattern-item:last-child {
border-bottom: none;
}
.analytics-pattern-item .pattern-main {
display: flex;
justify-content: space-between;
align-items: center;
gap: 8px;
}
.analytics-pattern-item .pattern-mode {
font-size: 10px;
font-weight: 600;
color: var(--text-primary, #e0e6ed);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.analytics-pattern-item .pattern-device {
font-size: 10px;
color: var(--text-dim, #5a6a7a);
font-family: var(--font-mono, monospace);
}
.analytics-pattern-item .pattern-meta {
display: flex;
gap: 10px;
font-size: 10px;
color: var(--text-dim, #5a6a7a);
flex-wrap: wrap;
}
.analytics-pattern-item .pattern-confidence {
color: var(--accent-green, #38c180);
font-weight: 600;
}
/* Geofence zone list */ /* Geofence zone list */
.geofence-zone-item { .geofence-zone-item {
+145 -30
View File
@@ -21,21 +21,25 @@ const Analytics = (function () {
} }
} }
function refresh() { function refresh() {
Promise.all([ Promise.all([
fetch('/analytics/summary').then(r => r.json()).catch(() => null), fetch('/analytics/summary').then(r => r.json()).catch(() => null),
fetch('/analytics/activity').then(r => r.json()).catch(() => null), fetch('/analytics/activity').then(r => r.json()).catch(() => null),
fetch('/alerts/events?limit=20').then(r => r.json()).catch(() => null), fetch('/analytics/insights').then(r => r.json()).catch(() => null),
fetch('/correlation').then(r => r.json()).catch(() => null), fetch('/analytics/patterns').then(r => r.json()).catch(() => null),
fetch('/analytics/geofences').then(r => r.json()).catch(() => null), fetch('/alerts/events?limit=20').then(r => r.json()).catch(() => null),
]).then(([summary, activity, alerts, correlations, geofences]) => { fetch('/correlation').then(r => r.json()).catch(() => null),
if (summary) renderSummary(summary); fetch('/analytics/geofences').then(r => r.json()).catch(() => null),
if (activity) renderSparklines(activity.sparklines || {}); ]).then(([summary, activity, insights, patterns, alerts, correlations, geofences]) => {
if (alerts) renderAlerts(alerts.events || []); if (summary) renderSummary(summary);
if (correlations) renderCorrelations(correlations); if (activity) renderSparklines(activity.sparklines || {});
if (geofences) renderGeofences(geofences.zones || []); if (insights) renderInsights(insights);
}); if (patterns) renderPatterns(patterns.patterns || []);
} if (alerts) renderAlerts(alerts.events || []);
if (correlations) renderCorrelations(correlations);
if (geofences) renderGeofences(geofences.zones || []);
});
}
function renderSummary(data) { function renderSummary(data) {
const counts = data.counts || {}; const counts = data.counts || {};
@@ -85,14 +89,18 @@ const Analytics = (function () {
} }
} }
function renderSparklines(sparklines) { function renderSparklines(sparklines) {
const map = { const map = {
adsb: 'analyticsSparkAdsb', adsb: 'analyticsSparkAdsb',
ais: 'analyticsSparkAis', ais: 'analyticsSparkAis',
wifi: 'analyticsSparkWifi', wifi: 'analyticsSparkWifi',
bluetooth: 'analyticsSparkBt', bluetooth: 'analyticsSparkBt',
dsc: 'analyticsSparkDsc', dsc: 'analyticsSparkDsc',
}; acars: 'analyticsSparkAcars',
vdl2: 'analyticsSparkVdl2',
aprs: 'analyticsSparkAprs',
meshtastic: 'analyticsSparkMesh',
};
for (const [mode, elId] of Object.entries(map)) { for (const [mode, elId] of Object.entries(map)) {
const el = document.getElementById(elId); const el = document.getElementById(elId);
@@ -109,9 +117,101 @@ const Analytics = (function () {
const points = data.map((v, i) => const points = data.map((v, i) =>
(i * step).toFixed(1) + ',' + (h - (v / max) * (h - 2)).toFixed(1) (i * step).toFixed(1) + ',' + (h - (v / max) * (h - 2)).toFixed(1)
).join(' '); ).join(' ');
el.innerHTML = '<svg viewBox="0 0 ' + w + ' ' + h + '" preserveAspectRatio="none"><polyline points="' + points + '"/></svg>'; el.innerHTML = '<svg viewBox="0 0 ' + w + ' ' + h + '" preserveAspectRatio="none"><polyline points="' + points + '"/></svg>';
} }
} }
function renderInsights(data) {
const cards = data.cards || [];
const topChanges = data.top_changes || [];
const cardsEl = document.getElementById('analyticsInsights');
const changesEl = document.getElementById('analyticsTopChanges');
if (cardsEl) {
if (!cards.length) {
cardsEl.innerHTML = '<div class="analytics-empty">No insight data available</div>';
} else {
cardsEl.innerHTML = cards.map(c => {
const sev = _esc(c.severity || 'low');
const title = _esc(c.title || 'Insight');
const value = _esc(c.value || '--');
const label = _esc(c.label || '');
const detail = _esc(c.detail || '');
return '<div class="analytics-insight-card ' + sev + '">' +
'<div class="insight-title">' + title + '</div>' +
'<div class="insight-value">' + value + '</div>' +
'<div class="insight-label">' + label + '</div>' +
'<div class="insight-detail">' + detail + '</div>' +
'</div>';
}).join('');
}
}
if (changesEl) {
if (!topChanges.length) {
changesEl.innerHTML = '<div class="analytics-empty">No change signals yet</div>';
} else {
changesEl.innerHTML = topChanges.map(item => {
const mode = _esc(item.mode_label || item.mode || '');
const deltaRaw = Number(item.delta || 0);
const trendClass = deltaRaw > 0 ? 'up' : (deltaRaw < 0 ? 'down' : 'flat');
const delta = _esc(item.signed_delta || String(deltaRaw));
const recentAvg = _esc(item.recent_avg);
const prevAvg = _esc(item.previous_avg);
return '<div class="analytics-change-row">' +
'<span class="mode">' + mode + '</span>' +
'<span class="delta ' + trendClass + '">' + delta + '</span>' +
'<span class="avg">avg ' + recentAvg + ' vs ' + prevAvg + '</span>' +
'</div>';
}).join('');
}
}
}
function renderPatterns(patterns) {
const container = document.getElementById('analyticsPatternList');
if (!container) return;
if (!patterns || patterns.length === 0) {
container.innerHTML = '<div class="analytics-empty">No recurring patterns detected</div>';
return;
}
const modeLabels = {
adsb: 'ADS-B',
ais: 'AIS',
wifi: 'WiFi',
bluetooth: 'Bluetooth',
dsc: 'DSC',
acars: 'ACARS',
vdl2: 'VDL2',
aprs: 'APRS',
meshtastic: 'Meshtastic',
};
const sorted = patterns
.slice()
.sort((a, b) => (b.confidence || 0) - (a.confidence || 0))
.slice(0, 20);
container.innerHTML = sorted.map(p => {
const confidencePct = Math.round((Number(p.confidence || 0)) * 100);
const mode = modeLabels[p.mode] || (p.mode || '--').toUpperCase();
const period = _humanPeriod(Number(p.period_seconds || 0));
const occurrences = Number(p.occurrences || 0);
const deviceId = _shortId(p.device_id || '--');
return '<div class="analytics-pattern-item">' +
'<div class="pattern-main">' +
'<span class="pattern-mode">' + _esc(mode) + '</span>' +
'<span class="pattern-device">' + _esc(deviceId) + '</span>' +
'</div>' +
'<div class="pattern-meta">' +
'<span>Period: ' + _esc(period) + '</span>' +
'<span>Hits: ' + _esc(occurrences) + '</span>' +
'<span class="pattern-confidence">' + _esc(confidencePct) + '%</span>' +
'</div>' +
'</div>';
}).join('');
}
function renderAlerts(events) { function renderAlerts(events) {
const container = document.getElementById('analyticsAlertFeed'); const container = document.getElementById('analyticsAlertFeed');
@@ -206,10 +306,25 @@ const Analytics = (function () {
if (el) el.textContent = val; if (el) el.textContent = val;
} }
function _esc(s) { function _esc(s) {
if (typeof s !== 'string') s = String(s == null ? '' : s); if (typeof s !== 'string') s = String(s == null ? '' : s);
return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;'); return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
} }
function _shortId(value) {
const text = String(value || '');
if (text.length <= 18) return text;
return text.slice(0, 8) + '...' + text.slice(-6);
}
function _humanPeriod(seconds) {
if (!isFinite(seconds) || seconds <= 0) return '--';
if (seconds < 60) return Math.round(seconds) + 's';
const mins = seconds / 60;
if (mins < 60) return mins.toFixed(mins < 10 ? 1 : 0) + 'm';
const hours = mins / 60;
return hours.toFixed(hours < 10 ? 1 : 0) + 'h';
}
return { init, destroy, refresh, addGeofence, deleteGeofence, exportData }; return { init, destroy, refresh, addGeofence, deleteGeofence, exportData };
})(); })();
+48 -18
View File
@@ -2,10 +2,10 @@
<div id="analyticsMode" class="mode-content"> <div id="analyticsMode" class="mode-content">
{# Analytics Dashboard Sidebar Panel #} {# Analytics Dashboard Sidebar Panel #}
<div class="section"> <div class="section">
<h3 class="section-header collapsible" onclick="toggleSection(this)"> <h3 class="section-header collapsible" onclick="toggleSection(this)">
<span>Summary</span> <span>Summary</span>
<span class="collapse-icon">&#9660;</span> <span class="collapse-icon">&#9660;</span>
</h3> </h3>
<div class="section-content"> <div class="section-content">
<div class="analytics-grid" id="analyticsSummaryCards"> <div class="analytics-grid" id="analyticsSummaryCards">
@@ -55,13 +55,31 @@
<div class="card-sparkline" id="analyticsSparkMesh"></div> <div class="card-sparkline" id="analyticsSparkMesh"></div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<div class="section"> <div class="section">
<h3 class="section-header collapsible" onclick="toggleSection(this)"> <h3 class="section-header collapsible" onclick="toggleSection(this)">
<span>Mode Health</span> <span>Operational Insights</span>
<span class="collapse-icon">&#9660;</span> <span class="collapse-icon">&#9660;</span>
</h3>
<div class="section-content">
<div class="analytics-insight-grid" id="analyticsInsights">
<div class="analytics-empty">Insights loading...</div>
</div>
<div class="analytics-top-changes">
<div class="analytics-section-header">Top Changes</div>
<div id="analyticsTopChanges">
<div class="analytics-empty">No change signals yet</div>
</div>
</div>
</div>
</div>
<div class="section">
<h3 class="section-header collapsible" onclick="toggleSection(this)">
<span>Mode Health</span>
<span class="collapse-icon">&#9660;</span>
</h3> </h3>
<div class="section-content"> <div class="section-content">
<div class="analytics-health" id="analyticsHealth"></div> <div class="analytics-health" id="analyticsHealth"></div>
@@ -77,13 +95,25 @@
<div class="squawk-emergency" id="analyticsSquawkPanel"> <div class="squawk-emergency" id="analyticsSquawkPanel">
<div class="squawk-title">Active Emergency Codes</div> <div class="squawk-title">Active Emergency Codes</div>
<div id="analyticsSquawkList"></div> <div id="analyticsSquawkList"></div>
</div> </div>
</div> </div>
</div> </div>
<div class="section"> <div class="section">
<h3 class="section-header collapsible" onclick="toggleSection(this)"> <h3 class="section-header collapsible" onclick="toggleSection(this)">
<span>Recent Alerts</span> <span>Temporal Patterns</span>
<span class="collapse-icon">&#9660;</span>
</h3>
<div class="section-content">
<div id="analyticsPatternList">
<div class="analytics-empty">No recurring patterns detected</div>
</div>
</div>
</div>
<div class="section">
<h3 class="section-header collapsible" onclick="toggleSection(this)">
<span>Recent Alerts</span>
<span class="collapse-icon">&#9660;</span> <span class="collapse-icon">&#9660;</span>
</h3> </h3>
<div class="section-content"> <div class="section-content">
+45
View File
@@ -6,16 +6,42 @@ from typing import Any
from utils.alerts import get_alert_manager from utils.alerts import get_alert_manager
from utils.recording import get_recording_manager from utils.recording import get_recording_manager
from utils.temporal_patterns import get_pattern_detector
IGNORE_TYPES = {'keepalive', 'ping'} IGNORE_TYPES = {'keepalive', 'ping'}
DEVICE_ID_FIELDS = (
'device_id',
'id',
'mac',
'mac_address',
'address',
'bssid',
'station_mac',
'client_mac',
'icao',
'callsign',
'mmsi',
'uuid',
'hash',
)
def process_event(mode: str, event: dict | Any, event_type: str | None = None) -> None: def process_event(mode: str, event: dict | Any, event_type: str | None = None) -> None:
if event_type in IGNORE_TYPES: if event_type in IGNORE_TYPES:
return return
if not isinstance(event, dict): if not isinstance(event, dict):
return return
device_id = _extract_device_id(event)
if device_id:
try:
get_pattern_detector().record_event(device_id=device_id, mode=mode)
except Exception:
# Pattern tracking should not break ingest pipeline
pass
try: try:
get_recording_manager().record_event(mode, event, event_type) get_recording_manager().record_event(mode, event, event_type)
except Exception: except Exception:
@@ -27,3 +53,22 @@ def process_event(mode: str, event: dict | Any, event_type: str | None = None) -
except Exception: except Exception:
# Alert failures should never break streaming # Alert failures should never break streaming
pass pass
def _extract_device_id(event: dict) -> str | None:
for field in DEVICE_ID_FIELDS:
value = event.get(field)
if value is None:
continue
text = str(value).strip()
if text:
return text
nested_candidates = ('target', 'device', 'source', 'aircraft', 'vessel')
for key in nested_candidates:
nested = event.get(key)
if isinstance(nested, dict):
nested_id = _extract_device_id(nested)
if nested_id:
return nested_id
return None