Merge branch 'pr-124'

# Conflicts:
#	app.py
#	routes/__init__.py
#	utils/database.py
This commit is contained in:
Smittix
2026-02-08 15:04:17 +00:00
14 changed files with 5184 additions and 263 deletions

View File

@@ -88,65 +88,111 @@ def init_db() -> None:
ON signal_history(mode, device_id, timestamp)
''')
# Device correlation table
conn.execute('''
CREATE TABLE IF NOT EXISTS device_correlations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
wifi_mac TEXT,
bt_mac TEXT,
confidence REAL,
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
UNIQUE(wifi_mac, bt_mac)
)
''')
# Alert rules
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
mode TEXT,
event_type TEXT,
match TEXT,
severity TEXT DEFAULT 'medium',
enabled BOOLEAN DEFAULT 1,
notify TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# Alert events
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id INTEGER,
mode TEXT,
event_type TEXT,
severity TEXT DEFAULT 'medium',
title TEXT,
message TEXT,
payload TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
)
''')
# Session recordings
conn.execute('''
CREATE TABLE IF NOT EXISTS recording_sessions (
id TEXT PRIMARY KEY,
mode TEXT NOT NULL,
label TEXT,
started_at TIMESTAMP NOT NULL,
stopped_at TIMESTAMP,
file_path TEXT NOT NULL,
event_count INTEGER DEFAULT 0,
size_bytes INTEGER DEFAULT 0,
metadata TEXT
)
''')
# Device correlation table
conn.execute('''
CREATE TABLE IF NOT EXISTS device_correlations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
wifi_mac TEXT,
bt_mac TEXT,
confidence REAL,
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
UNIQUE(wifi_mac, bt_mac)
)
''')
# Alert rules
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
mode TEXT,
event_type TEXT,
match TEXT,
severity TEXT DEFAULT 'medium',
enabled BOOLEAN DEFAULT 1,
notify TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# Alert events
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id INTEGER,
mode TEXT,
event_type TEXT,
severity TEXT DEFAULT 'medium',
title TEXT,
message TEXT,
payload TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
)
''')
# Session recordings
conn.execute('''
CREATE TABLE IF NOT EXISTS recording_sessions (
id TEXT PRIMARY KEY,
mode TEXT NOT NULL,
label TEXT,
started_at TIMESTAMP NOT NULL,
stopped_at TIMESTAMP,
file_path TEXT NOT NULL,
event_count INTEGER DEFAULT 0,
size_bytes INTEGER DEFAULT 0,
metadata TEXT
)
''')
# Alert rules
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
mode TEXT,
event_type TEXT,
match TEXT,
severity TEXT DEFAULT 'medium',
enabled BOOLEAN DEFAULT 1,
notify TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# Alert events
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id INTEGER,
mode TEXT,
event_type TEXT,
severity TEXT DEFAULT 'medium',
title TEXT,
message TEXT,
payload TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
)
''')
# Session recordings
conn.execute('''
CREATE TABLE IF NOT EXISTS recording_sessions (
id TEXT PRIMARY KEY,
mode TEXT NOT NULL,
label TEXT,
started_at TIMESTAMP NOT NULL,
stopped_at TIMESTAMP,
file_path TEXT NOT NULL,
event_count INTEGER DEFAULT 0,
size_bytes INTEGER DEFAULT 0,
metadata TEXT
)
''')
# Users table for authentication
conn.execute('''
@@ -177,29 +223,29 @@ def init_db() -> None:
# =====================================================================
# TSCM Baselines - Environment snapshots for comparison
conn.execute('''
CREATE TABLE IF NOT EXISTS tscm_baselines (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
location TEXT,
description TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
wifi_networks TEXT,
wifi_clients TEXT,
bt_devices TEXT,
rf_frequencies TEXT,
gps_coords TEXT,
is_active BOOLEAN DEFAULT 0
)
''')
# Ensure new columns exist for older databases
try:
columns = {row['name'] for row in conn.execute("PRAGMA table_info(tscm_baselines)")}
if 'wifi_clients' not in columns:
conn.execute('ALTER TABLE tscm_baselines ADD COLUMN wifi_clients TEXT')
except Exception as e:
logger.debug(f"Schema update skipped for tscm_baselines: {e}")
conn.execute('''
CREATE TABLE IF NOT EXISTS tscm_baselines (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
location TEXT,
description TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
wifi_networks TEXT,
wifi_clients TEXT,
bt_devices TEXT,
rf_frequencies TEXT,
gps_coords TEXT,
is_active BOOLEAN DEFAULT 0
)
''')
# Ensure new columns exist for older databases
try:
columns = {row['name'] for row in conn.execute("PRAGMA table_info(tscm_baselines)")}
if 'wifi_clients' not in columns:
conn.execute('ALTER TABLE tscm_baselines ADD COLUMN wifi_clients TEXT')
except Exception as e:
logger.debug(f"Schema update skipped for tscm_baselines: {e}")
# TSCM Sweeps - Individual sweep sessions
conn.execute('''
@@ -407,6 +453,134 @@ def init_db() -> None:
ON tscm_cases(status, created_at)
''')
# =====================================================================
# GSM (Global System for Mobile) Intelligence Tables
# =====================================================================
# gsm_cells - Known cell towers (OpenCellID cache)
conn.execute('''
CREATE TABLE IF NOT EXISTS gsm_cells (
id INTEGER PRIMARY KEY AUTOINCREMENT,
mcc INTEGER NOT NULL,
mnc INTEGER NOT NULL,
lac INTEGER NOT NULL,
cid INTEGER NOT NULL,
lat REAL,
lon REAL,
azimuth INTEGER,
range_meters INTEGER,
samples INTEGER,
radio TEXT,
operator TEXT,
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_verified TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
UNIQUE(mcc, mnc, lac, cid)
)
''')
# gsm_rogues - Detected rogue towers / IMSI catchers
conn.execute('''
CREATE TABLE IF NOT EXISTS gsm_rogues (
id INTEGER PRIMARY KEY AUTOINCREMENT,
arfcn INTEGER NOT NULL,
mcc INTEGER,
mnc INTEGER,
lac INTEGER,
cid INTEGER,
signal_strength REAL,
reason TEXT NOT NULL,
threat_level TEXT DEFAULT 'medium',
detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
location_lat REAL,
location_lon REAL,
acknowledged BOOLEAN DEFAULT 0,
notes TEXT,
metadata TEXT
)
''')
# gsm_signals - 60-day archive of signal observations
conn.execute('''
CREATE TABLE IF NOT EXISTS gsm_signals (
id INTEGER PRIMARY KEY AUTOINCREMENT,
imsi TEXT,
tmsi TEXT,
mcc INTEGER,
mnc INTEGER,
lac INTEGER,
cid INTEGER,
ta_value INTEGER,
signal_strength REAL,
arfcn INTEGER,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
metadata TEXT
)
''')
# gsm_tmsi_log - 24-hour raw pings for crowd density
conn.execute('''
CREATE TABLE IF NOT EXISTS gsm_tmsi_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tmsi TEXT NOT NULL,
lac INTEGER,
cid INTEGER,
ta_value INTEGER,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# gsm_velocity_log - 1-hour buffer for movement tracking
conn.execute('''
CREATE TABLE IF NOT EXISTS gsm_velocity_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
device_id TEXT NOT NULL,
prev_ta INTEGER,
curr_ta INTEGER,
prev_cid INTEGER,
curr_cid INTEGER,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
estimated_velocity REAL,
metadata TEXT
)
''')
# GSM indexes for performance
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_gsm_cells_location
ON gsm_cells(lat, lon)
''')
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_gsm_cells_identity
ON gsm_cells(mcc, mnc, lac, cid)
''')
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_gsm_rogues_severity
ON gsm_rogues(threat_level, detected_at)
''')
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_gsm_signals_cell_time
ON gsm_signals(cid, lac, timestamp)
''')
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_gsm_signals_device
ON gsm_signals(imsi, tmsi, timestamp)
''')
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_gsm_tmsi_log_time
ON gsm_tmsi_log(timestamp)
''')
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_gsm_velocity_log_device
ON gsm_velocity_log(device_id, timestamp)
''')
# =====================================================================
# DSC (Digital Selective Calling) Tables
# =====================================================================
@@ -740,16 +914,16 @@ def get_correlations(min_confidence: float = 0.5) -> list[dict]:
# TSCM Functions
# =============================================================================
def create_tscm_baseline(
name: str,
location: str | None = None,
description: str | None = None,
wifi_networks: list | None = None,
wifi_clients: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None,
gps_coords: dict | None = None
) -> int:
def create_tscm_baseline(
name: str,
location: str | None = None,
description: str | None = None,
wifi_networks: list | None = None,
wifi_clients: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None,
gps_coords: dict | None = None
) -> int:
"""
Create a new TSCM baseline.
@@ -757,20 +931,20 @@ def create_tscm_baseline(
The ID of the created baseline
"""
with get_db() as conn:
cursor = conn.execute('''
INSERT INTO tscm_baselines
(name, location, description, wifi_networks, wifi_clients, bt_devices, rf_frequencies, gps_coords)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
name,
location,
description,
json.dumps(wifi_networks) if wifi_networks else None,
json.dumps(wifi_clients) if wifi_clients else None,
json.dumps(bt_devices) if bt_devices else None,
json.dumps(rf_frequencies) if rf_frequencies else None,
json.dumps(gps_coords) if gps_coords else None
))
cursor = conn.execute('''
INSERT INTO tscm_baselines
(name, location, description, wifi_networks, wifi_clients, bt_devices, rf_frequencies, gps_coords)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
name,
location,
description,
json.dumps(wifi_networks) if wifi_networks else None,
json.dumps(wifi_clients) if wifi_clients else None,
json.dumps(bt_devices) if bt_devices else None,
json.dumps(rf_frequencies) if rf_frequencies else None,
json.dumps(gps_coords) if gps_coords else None
))
return cursor.lastrowid
@@ -785,19 +959,19 @@ def get_tscm_baseline(baseline_id: int) -> dict | None:
if row is None:
return None
return {
'id': row['id'],
'name': row['name'],
'location': row['location'],
'description': row['description'],
'created_at': row['created_at'],
'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
'wifi_clients': json.loads(row['wifi_clients']) if row['wifi_clients'] else [],
'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
'is_active': bool(row['is_active'])
}
return {
'id': row['id'],
'name': row['name'],
'location': row['location'],
'description': row['description'],
'created_at': row['created_at'],
'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
'wifi_clients': json.loads(row['wifi_clients']) if row['wifi_clients'] else [],
'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
'is_active': bool(row['is_active'])
}
def get_all_tscm_baselines() -> list[dict]:
@@ -839,23 +1013,23 @@ def set_active_tscm_baseline(baseline_id: int) -> bool:
return cursor.rowcount > 0
def update_tscm_baseline(
baseline_id: int,
wifi_networks: list | None = None,
wifi_clients: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None
) -> bool:
def update_tscm_baseline(
baseline_id: int,
wifi_networks: list | None = None,
wifi_clients: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None
) -> bool:
"""Update baseline device lists."""
updates = []
params = []
if wifi_networks is not None:
updates.append('wifi_networks = ?')
params.append(json.dumps(wifi_networks))
if wifi_clients is not None:
updates.append('wifi_clients = ?')
params.append(json.dumps(wifi_clients))
if wifi_networks is not None:
updates.append('wifi_networks = ?')
params.append(json.dumps(wifi_networks))
if wifi_clients is not None:
updates.append('wifi_clients = ?')
params.append(json.dumps(wifi_clients))
if bt_devices is not None:
updates.append('bt_devices = ?')
params.append(json.dumps(bt_devices))
@@ -1267,127 +1441,127 @@ def get_all_known_devices(
]
def delete_known_device(identifier: str) -> bool:
"""Remove a device from the known-good registry."""
with get_db() as conn:
cursor = conn.execute(
'DELETE FROM tscm_known_devices WHERE identifier = ?',
(identifier.upper(),)
)
return cursor.rowcount > 0
# =============================================================================
# TSCM Schedule Functions
# =============================================================================
def create_tscm_schedule(
name: str,
cron_expression: str,
sweep_type: str = 'standard',
baseline_id: int | None = None,
zone_name: str | None = None,
enabled: bool = True,
notify_on_threat: bool = True,
notify_email: str | None = None,
last_run: str | None = None,
next_run: str | None = None,
) -> int:
"""Create a new TSCM sweep schedule."""
with get_db() as conn:
cursor = conn.execute('''
INSERT INTO tscm_schedules
(name, baseline_id, zone_name, cron_expression, sweep_type,
enabled, last_run, next_run, notify_on_threat, notify_email)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
name,
baseline_id,
zone_name,
cron_expression,
sweep_type,
1 if enabled else 0,
last_run,
next_run,
1 if notify_on_threat else 0,
notify_email,
))
return cursor.lastrowid
def get_tscm_schedule(schedule_id: int) -> dict | None:
"""Get a TSCM schedule by ID."""
with get_db() as conn:
cursor = conn.execute(
'SELECT * FROM tscm_schedules WHERE id = ?',
(schedule_id,)
)
row = cursor.fetchone()
return dict(row) if row else None
def get_all_tscm_schedules(
enabled: bool | None = None,
limit: int = 200
) -> list[dict]:
"""Get all TSCM schedules."""
conditions = []
params = []
if enabled is not None:
conditions.append('enabled = ?')
params.append(1 if enabled else 0)
where_clause = f'WHERE {" AND ".join(conditions)}' if conditions else ''
params.append(limit)
with get_db() as conn:
cursor = conn.execute(f'''
SELECT * FROM tscm_schedules
{where_clause}
ORDER BY id DESC
LIMIT ?
''', params)
return [dict(row) for row in cursor]
def update_tscm_schedule(schedule_id: int, **fields) -> bool:
"""Update a TSCM schedule."""
if not fields:
return False
updates = []
params = []
for key, value in fields.items():
updates.append(f'{key} = ?')
params.append(value)
params.append(schedule_id)
with get_db() as conn:
cursor = conn.execute(
f'UPDATE tscm_schedules SET {", ".join(updates)} WHERE id = ?',
params
)
return cursor.rowcount > 0
def delete_tscm_schedule(schedule_id: int) -> bool:
"""Delete a TSCM schedule."""
with get_db() as conn:
cursor = conn.execute(
'DELETE FROM tscm_schedules WHERE id = ?',
(schedule_id,)
)
return cursor.rowcount > 0
def is_known_good_device(identifier: str, location: str | None = None) -> dict | None:
"""Check if a device is in the known-good registry for a location."""
with get_db() as conn:
if location:
cursor = conn.execute('''
def delete_known_device(identifier: str) -> bool:
"""Remove a device from the known-good registry."""
with get_db() as conn:
cursor = conn.execute(
'DELETE FROM tscm_known_devices WHERE identifier = ?',
(identifier.upper(),)
)
return cursor.rowcount > 0
# =============================================================================
# TSCM Schedule Functions
# =============================================================================
def create_tscm_schedule(
name: str,
cron_expression: str,
sweep_type: str = 'standard',
baseline_id: int | None = None,
zone_name: str | None = None,
enabled: bool = True,
notify_on_threat: bool = True,
notify_email: str | None = None,
last_run: str | None = None,
next_run: str | None = None,
) -> int:
"""Create a new TSCM sweep schedule."""
with get_db() as conn:
cursor = conn.execute('''
INSERT INTO tscm_schedules
(name, baseline_id, zone_name, cron_expression, sweep_type,
enabled, last_run, next_run, notify_on_threat, notify_email)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
name,
baseline_id,
zone_name,
cron_expression,
sweep_type,
1 if enabled else 0,
last_run,
next_run,
1 if notify_on_threat else 0,
notify_email,
))
return cursor.lastrowid
def get_tscm_schedule(schedule_id: int) -> dict | None:
"""Get a TSCM schedule by ID."""
with get_db() as conn:
cursor = conn.execute(
'SELECT * FROM tscm_schedules WHERE id = ?',
(schedule_id,)
)
row = cursor.fetchone()
return dict(row) if row else None
def get_all_tscm_schedules(
enabled: bool | None = None,
limit: int = 200
) -> list[dict]:
"""Get all TSCM schedules."""
conditions = []
params = []
if enabled is not None:
conditions.append('enabled = ?')
params.append(1 if enabled else 0)
where_clause = f'WHERE {" AND ".join(conditions)}' if conditions else ''
params.append(limit)
with get_db() as conn:
cursor = conn.execute(f'''
SELECT * FROM tscm_schedules
{where_clause}
ORDER BY id DESC
LIMIT ?
''', params)
return [dict(row) for row in cursor]
def update_tscm_schedule(schedule_id: int, **fields) -> bool:
"""Update a TSCM schedule."""
if not fields:
return False
updates = []
params = []
for key, value in fields.items():
updates.append(f'{key} = ?')
params.append(value)
params.append(schedule_id)
with get_db() as conn:
cursor = conn.execute(
f'UPDATE tscm_schedules SET {", ".join(updates)} WHERE id = ?',
params
)
return cursor.rowcount > 0
def delete_tscm_schedule(schedule_id: int) -> bool:
"""Delete a TSCM schedule."""
with get_db() as conn:
cursor = conn.execute(
'DELETE FROM tscm_schedules WHERE id = ?',
(schedule_id,)
)
return cursor.rowcount > 0
def is_known_good_device(identifier: str, location: str | None = None) -> dict | None:
"""Check if a device is in the known-good registry for a location."""
with get_db() as conn:
if location:
cursor = conn.execute('''
SELECT * FROM tscm_known_devices
WHERE identifier = ? AND (location = ? OR scope = 'global')
''', (identifier.upper(), location))
@@ -2123,3 +2297,61 @@ def cleanup_old_payloads(max_age_hours: int = 24) -> int:
WHERE received_at < datetime('now', ?)
''', (f'-{max_age_hours} hours',))
return cursor.rowcount
# =============================================================================
# GSM Cleanup Functions
# =============================================================================
def cleanup_old_gsm_signals(max_age_days: int = 60) -> int:
"""
Remove old GSM signal observations (60-day archive).
Args:
max_age_days: Maximum age in days (default: 60)
Returns:
Number of deleted entries
"""
with get_db() as conn:
cursor = conn.execute('''
DELETE FROM gsm_signals
WHERE timestamp < datetime('now', ?)
''', (f'-{max_age_days} days',))
return cursor.rowcount
def cleanup_old_gsm_tmsi_log(max_age_hours: int = 24) -> int:
"""
Remove old TMSI log entries (24-hour buffer for crowd density).
Args:
max_age_hours: Maximum age in hours (default: 24)
Returns:
Number of deleted entries
"""
with get_db() as conn:
cursor = conn.execute('''
DELETE FROM gsm_tmsi_log
WHERE timestamp < datetime('now', ?)
''', (f'-{max_age_hours} hours',))
return cursor.rowcount
def cleanup_old_gsm_velocity_log(max_age_hours: int = 1) -> int:
"""
Remove old velocity log entries (1-hour buffer for movement tracking).
Args:
max_age_hours: Maximum age in hours (default: 1)
Returns:
Number of deleted entries
"""
with get_db() as conn:
cursor = conn.execute('''
DELETE FROM gsm_velocity_log
WHERE timestamp < datetime('now', ?)
''', (f'-{max_age_hours} hours',))
return cursor.rowcount