mirror of
https://github.com/smittix/intercept.git
synced 2026-06-21 19:51:05 -07:00
Merge upstream/main: add gsm_spy blueprint
This commit is contained in:
+30
-2
@@ -142,7 +142,7 @@ class DataStore:
|
||||
|
||||
|
||||
class CleanupManager:
|
||||
"""Manages periodic cleanup of multiple data stores."""
|
||||
"""Manages periodic cleanup of multiple data stores and database tables."""
|
||||
|
||||
def __init__(self, interval: float = 60.0):
|
||||
"""
|
||||
@@ -152,9 +152,11 @@ class CleanupManager:
|
||||
interval: Cleanup interval in seconds
|
||||
"""
|
||||
self.stores: list[DataStore] = []
|
||||
self.db_cleanup_funcs: list[tuple[callable, int]] = [] # (func, interval_multiplier)
|
||||
self.interval = interval
|
||||
self._timer: threading.Timer | None = None
|
||||
self._running = False
|
||||
self._cleanup_count = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def register(self, store: DataStore) -> None:
|
||||
@@ -169,6 +171,17 @@ class CleanupManager:
|
||||
if store in self.stores:
|
||||
self.stores.remove(store)
|
||||
|
||||
def register_db_cleanup(self, func: callable, interval_multiplier: int = 60) -> None:
|
||||
"""
|
||||
Register a database cleanup function.
|
||||
|
||||
Args:
|
||||
func: Cleanup function to call (should return number of deleted rows)
|
||||
interval_multiplier: How many cleanup cycles to wait between calls (default: 60 = 1 hour if interval is 60s)
|
||||
"""
|
||||
with self._lock:
|
||||
self.db_cleanup_funcs.append((func, interval_multiplier))
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the cleanup timer."""
|
||||
with self._lock:
|
||||
@@ -194,11 +207,15 @@ class CleanupManager:
|
||||
self._timer.start()
|
||||
|
||||
def _run_cleanup(self) -> None:
|
||||
"""Run cleanup on all registered stores."""
|
||||
"""Run cleanup on all registered stores and database tables."""
|
||||
total_cleaned = 0
|
||||
|
||||
# Cleanup in-memory data stores
|
||||
with self._lock:
|
||||
stores = list(self.stores)
|
||||
db_funcs = list(self.db_cleanup_funcs)
|
||||
self._cleanup_count += 1
|
||||
current_count = self._cleanup_count
|
||||
|
||||
for store in stores:
|
||||
try:
|
||||
@@ -206,6 +223,17 @@ class CleanupManager:
|
||||
except Exception as e:
|
||||
logger.error(f"Error cleaning up {store.name}: {e}")
|
||||
|
||||
# Cleanup database tables (less frequently)
|
||||
for func, interval_multiplier in db_funcs:
|
||||
if current_count % interval_multiplier == 0:
|
||||
try:
|
||||
deleted = func()
|
||||
if deleted > 0:
|
||||
logger.info(f"Database cleanup: {func.__name__} removed {deleted} rows")
|
||||
total_cleaned += deleted
|
||||
except Exception as e:
|
||||
logger.error(f"Error in database cleanup {func.__name__}: {e}")
|
||||
|
||||
if total_cleaned > 0:
|
||||
logger.info(f"Cleanup complete: removed {total_cleaned} stale entries")
|
||||
|
||||
|
||||
@@ -274,3 +274,14 @@ MAX_DEAUTH_ALERTS_AGE_SECONDS = 300 # 5 minutes
|
||||
|
||||
# Deauth detector sniff timeout (seconds)
|
||||
DEAUTH_SNIFF_TIMEOUT = 0.5
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# GSM SPY (Cellular Intelligence)
|
||||
# =============================================================================
|
||||
|
||||
# Maximum age for GSM tower/device data in DataStore (seconds)
|
||||
MAX_GSM_AGE_SECONDS = 300 # 5 minutes
|
||||
|
||||
# Timing Advance conversion to meters
|
||||
GSM_TA_METERS_PER_UNIT = 554
|
||||
|
||||
+485
-253
@@ -88,65 +88,111 @@ def init_db() -> None:
|
||||
ON signal_history(mode, device_id, timestamp)
|
||||
''')
|
||||
|
||||
# Device correlation table
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS device_correlations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
wifi_mac TEXT,
|
||||
bt_mac TEXT,
|
||||
confidence REAL,
|
||||
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
UNIQUE(wifi_mac, bt_mac)
|
||||
)
|
||||
''')
|
||||
|
||||
# Alert rules
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS alert_rules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
mode TEXT,
|
||||
event_type TEXT,
|
||||
match TEXT,
|
||||
severity TEXT DEFAULT 'medium',
|
||||
enabled BOOLEAN DEFAULT 1,
|
||||
notify TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# Alert events
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS alert_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
rule_id INTEGER,
|
||||
mode TEXT,
|
||||
event_type TEXT,
|
||||
severity TEXT DEFAULT 'medium',
|
||||
title TEXT,
|
||||
message TEXT,
|
||||
payload TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
|
||||
)
|
||||
''')
|
||||
|
||||
# Session recordings
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS recording_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
mode TEXT NOT NULL,
|
||||
label TEXT,
|
||||
started_at TIMESTAMP NOT NULL,
|
||||
stopped_at TIMESTAMP,
|
||||
file_path TEXT NOT NULL,
|
||||
event_count INTEGER DEFAULT 0,
|
||||
size_bytes INTEGER DEFAULT 0,
|
||||
metadata TEXT
|
||||
)
|
||||
''')
|
||||
# Device correlation table
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS device_correlations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
wifi_mac TEXT,
|
||||
bt_mac TEXT,
|
||||
confidence REAL,
|
||||
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
UNIQUE(wifi_mac, bt_mac)
|
||||
)
|
||||
''')
|
||||
|
||||
# Alert rules
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS alert_rules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
mode TEXT,
|
||||
event_type TEXT,
|
||||
match TEXT,
|
||||
severity TEXT DEFAULT 'medium',
|
||||
enabled BOOLEAN DEFAULT 1,
|
||||
notify TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# Alert events
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS alert_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
rule_id INTEGER,
|
||||
mode TEXT,
|
||||
event_type TEXT,
|
||||
severity TEXT DEFAULT 'medium',
|
||||
title TEXT,
|
||||
message TEXT,
|
||||
payload TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
|
||||
)
|
||||
''')
|
||||
|
||||
# Session recordings
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS recording_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
mode TEXT NOT NULL,
|
||||
label TEXT,
|
||||
started_at TIMESTAMP NOT NULL,
|
||||
stopped_at TIMESTAMP,
|
||||
file_path TEXT NOT NULL,
|
||||
event_count INTEGER DEFAULT 0,
|
||||
size_bytes INTEGER DEFAULT 0,
|
||||
metadata TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
# Alert rules
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS alert_rules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
mode TEXT,
|
||||
event_type TEXT,
|
||||
match TEXT,
|
||||
severity TEXT DEFAULT 'medium',
|
||||
enabled BOOLEAN DEFAULT 1,
|
||||
notify TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# Alert events
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS alert_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
rule_id INTEGER,
|
||||
mode TEXT,
|
||||
event_type TEXT,
|
||||
severity TEXT DEFAULT 'medium',
|
||||
title TEXT,
|
||||
message TEXT,
|
||||
payload TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
|
||||
)
|
||||
''')
|
||||
|
||||
# Session recordings
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS recording_sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
mode TEXT NOT NULL,
|
||||
label TEXT,
|
||||
started_at TIMESTAMP NOT NULL,
|
||||
stopped_at TIMESTAMP,
|
||||
file_path TEXT NOT NULL,
|
||||
event_count INTEGER DEFAULT 0,
|
||||
size_bytes INTEGER DEFAULT 0,
|
||||
metadata TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
# Users table for authentication
|
||||
conn.execute('''
|
||||
@@ -177,29 +223,29 @@ def init_db() -> None:
|
||||
# =====================================================================
|
||||
|
||||
# TSCM Baselines - Environment snapshots for comparison
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS tscm_baselines (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
location TEXT,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
wifi_networks TEXT,
|
||||
wifi_clients TEXT,
|
||||
bt_devices TEXT,
|
||||
rf_frequencies TEXT,
|
||||
gps_coords TEXT,
|
||||
is_active BOOLEAN DEFAULT 0
|
||||
)
|
||||
''')
|
||||
|
||||
# Ensure new columns exist for older databases
|
||||
try:
|
||||
columns = {row['name'] for row in conn.execute("PRAGMA table_info(tscm_baselines)")}
|
||||
if 'wifi_clients' not in columns:
|
||||
conn.execute('ALTER TABLE tscm_baselines ADD COLUMN wifi_clients TEXT')
|
||||
except Exception as e:
|
||||
logger.debug(f"Schema update skipped for tscm_baselines: {e}")
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS tscm_baselines (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
location TEXT,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
wifi_networks TEXT,
|
||||
wifi_clients TEXT,
|
||||
bt_devices TEXT,
|
||||
rf_frequencies TEXT,
|
||||
gps_coords TEXT,
|
||||
is_active BOOLEAN DEFAULT 0
|
||||
)
|
||||
''')
|
||||
|
||||
# Ensure new columns exist for older databases
|
||||
try:
|
||||
columns = {row['name'] for row in conn.execute("PRAGMA table_info(tscm_baselines)")}
|
||||
if 'wifi_clients' not in columns:
|
||||
conn.execute('ALTER TABLE tscm_baselines ADD COLUMN wifi_clients TEXT')
|
||||
except Exception as e:
|
||||
logger.debug(f"Schema update skipped for tscm_baselines: {e}")
|
||||
|
||||
# TSCM Sweeps - Individual sweep sessions
|
||||
conn.execute('''
|
||||
@@ -407,6 +453,134 @@ def init_db() -> None:
|
||||
ON tscm_cases(status, created_at)
|
||||
''')
|
||||
|
||||
# =====================================================================
|
||||
# GSM (Global System for Mobile) Intelligence Tables
|
||||
# =====================================================================
|
||||
|
||||
# gsm_cells - Known cell towers (OpenCellID cache)
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS gsm_cells (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
mcc INTEGER NOT NULL,
|
||||
mnc INTEGER NOT NULL,
|
||||
lac INTEGER NOT NULL,
|
||||
cid INTEGER NOT NULL,
|
||||
lat REAL,
|
||||
lon REAL,
|
||||
azimuth INTEGER,
|
||||
range_meters INTEGER,
|
||||
samples INTEGER,
|
||||
radio TEXT,
|
||||
operator TEXT,
|
||||
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
last_verified TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT,
|
||||
UNIQUE(mcc, mnc, lac, cid)
|
||||
)
|
||||
''')
|
||||
|
||||
# gsm_rogues - Detected rogue towers / IMSI catchers
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS gsm_rogues (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
arfcn INTEGER NOT NULL,
|
||||
mcc INTEGER,
|
||||
mnc INTEGER,
|
||||
lac INTEGER,
|
||||
cid INTEGER,
|
||||
signal_strength REAL,
|
||||
reason TEXT NOT NULL,
|
||||
threat_level TEXT DEFAULT 'medium',
|
||||
detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
location_lat REAL,
|
||||
location_lon REAL,
|
||||
acknowledged BOOLEAN DEFAULT 0,
|
||||
notes TEXT,
|
||||
metadata TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
# gsm_signals - 60-day archive of signal observations
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS gsm_signals (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
imsi TEXT,
|
||||
tmsi TEXT,
|
||||
mcc INTEGER,
|
||||
mnc INTEGER,
|
||||
lac INTEGER,
|
||||
cid INTEGER,
|
||||
ta_value INTEGER,
|
||||
signal_strength REAL,
|
||||
arfcn INTEGER,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
metadata TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
# gsm_tmsi_log - 24-hour raw pings for crowd density
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS gsm_tmsi_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tmsi TEXT NOT NULL,
|
||||
lac INTEGER,
|
||||
cid INTEGER,
|
||||
ta_value INTEGER,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# gsm_velocity_log - 1-hour buffer for movement tracking
|
||||
conn.execute('''
|
||||
CREATE TABLE IF NOT EXISTS gsm_velocity_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
device_id TEXT NOT NULL,
|
||||
prev_ta INTEGER,
|
||||
curr_ta INTEGER,
|
||||
prev_cid INTEGER,
|
||||
curr_cid INTEGER,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
estimated_velocity REAL,
|
||||
metadata TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
# GSM indexes for performance
|
||||
conn.execute('''
|
||||
CREATE INDEX IF NOT EXISTS idx_gsm_cells_location
|
||||
ON gsm_cells(lat, lon)
|
||||
''')
|
||||
|
||||
conn.execute('''
|
||||
CREATE INDEX IF NOT EXISTS idx_gsm_cells_identity
|
||||
ON gsm_cells(mcc, mnc, lac, cid)
|
||||
''')
|
||||
|
||||
conn.execute('''
|
||||
CREATE INDEX IF NOT EXISTS idx_gsm_rogues_severity
|
||||
ON gsm_rogues(threat_level, detected_at)
|
||||
''')
|
||||
|
||||
conn.execute('''
|
||||
CREATE INDEX IF NOT EXISTS idx_gsm_signals_cell_time
|
||||
ON gsm_signals(cid, lac, timestamp)
|
||||
''')
|
||||
|
||||
conn.execute('''
|
||||
CREATE INDEX IF NOT EXISTS idx_gsm_signals_device
|
||||
ON gsm_signals(imsi, tmsi, timestamp)
|
||||
''')
|
||||
|
||||
conn.execute('''
|
||||
CREATE INDEX IF NOT EXISTS idx_gsm_tmsi_log_time
|
||||
ON gsm_tmsi_log(timestamp)
|
||||
''')
|
||||
|
||||
conn.execute('''
|
||||
CREATE INDEX IF NOT EXISTS idx_gsm_velocity_log_device
|
||||
ON gsm_velocity_log(device_id, timestamp)
|
||||
''')
|
||||
|
||||
# =====================================================================
|
||||
# DSC (Digital Selective Calling) Tables
|
||||
# =====================================================================
|
||||
@@ -740,16 +914,16 @@ def get_correlations(min_confidence: float = 0.5) -> list[dict]:
|
||||
# TSCM Functions
|
||||
# =============================================================================
|
||||
|
||||
def create_tscm_baseline(
|
||||
name: str,
|
||||
location: str | None = None,
|
||||
description: str | None = None,
|
||||
wifi_networks: list | None = None,
|
||||
wifi_clients: list | None = None,
|
||||
bt_devices: list | None = None,
|
||||
rf_frequencies: list | None = None,
|
||||
gps_coords: dict | None = None
|
||||
) -> int:
|
||||
def create_tscm_baseline(
|
||||
name: str,
|
||||
location: str | None = None,
|
||||
description: str | None = None,
|
||||
wifi_networks: list | None = None,
|
||||
wifi_clients: list | None = None,
|
||||
bt_devices: list | None = None,
|
||||
rf_frequencies: list | None = None,
|
||||
gps_coords: dict | None = None
|
||||
) -> int:
|
||||
"""
|
||||
Create a new TSCM baseline.
|
||||
|
||||
@@ -757,20 +931,20 @@ def create_tscm_baseline(
|
||||
The ID of the created baseline
|
||||
"""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute('''
|
||||
INSERT INTO tscm_baselines
|
||||
(name, location, description, wifi_networks, wifi_clients, bt_devices, rf_frequencies, gps_coords)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
name,
|
||||
location,
|
||||
description,
|
||||
json.dumps(wifi_networks) if wifi_networks else None,
|
||||
json.dumps(wifi_clients) if wifi_clients else None,
|
||||
json.dumps(bt_devices) if bt_devices else None,
|
||||
json.dumps(rf_frequencies) if rf_frequencies else None,
|
||||
json.dumps(gps_coords) if gps_coords else None
|
||||
))
|
||||
cursor = conn.execute('''
|
||||
INSERT INTO tscm_baselines
|
||||
(name, location, description, wifi_networks, wifi_clients, bt_devices, rf_frequencies, gps_coords)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
name,
|
||||
location,
|
||||
description,
|
||||
json.dumps(wifi_networks) if wifi_networks else None,
|
||||
json.dumps(wifi_clients) if wifi_clients else None,
|
||||
json.dumps(bt_devices) if bt_devices else None,
|
||||
json.dumps(rf_frequencies) if rf_frequencies else None,
|
||||
json.dumps(gps_coords) if gps_coords else None
|
||||
))
|
||||
return cursor.lastrowid
|
||||
|
||||
|
||||
@@ -785,19 +959,19 @@ def get_tscm_baseline(baseline_id: int) -> dict | None:
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
return {
|
||||
'id': row['id'],
|
||||
'name': row['name'],
|
||||
'location': row['location'],
|
||||
'description': row['description'],
|
||||
'created_at': row['created_at'],
|
||||
'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
|
||||
'wifi_clients': json.loads(row['wifi_clients']) if row['wifi_clients'] else [],
|
||||
'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
|
||||
'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
|
||||
'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
|
||||
'is_active': bool(row['is_active'])
|
||||
}
|
||||
return {
|
||||
'id': row['id'],
|
||||
'name': row['name'],
|
||||
'location': row['location'],
|
||||
'description': row['description'],
|
||||
'created_at': row['created_at'],
|
||||
'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
|
||||
'wifi_clients': json.loads(row['wifi_clients']) if row['wifi_clients'] else [],
|
||||
'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
|
||||
'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
|
||||
'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
|
||||
'is_active': bool(row['is_active'])
|
||||
}
|
||||
|
||||
|
||||
def get_all_tscm_baselines() -> list[dict]:
|
||||
@@ -839,23 +1013,23 @@ def set_active_tscm_baseline(baseline_id: int) -> bool:
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
def update_tscm_baseline(
|
||||
baseline_id: int,
|
||||
wifi_networks: list | None = None,
|
||||
wifi_clients: list | None = None,
|
||||
bt_devices: list | None = None,
|
||||
rf_frequencies: list | None = None
|
||||
) -> bool:
|
||||
def update_tscm_baseline(
|
||||
baseline_id: int,
|
||||
wifi_networks: list | None = None,
|
||||
wifi_clients: list | None = None,
|
||||
bt_devices: list | None = None,
|
||||
rf_frequencies: list | None = None
|
||||
) -> bool:
|
||||
"""Update baseline device lists."""
|
||||
updates = []
|
||||
params = []
|
||||
|
||||
if wifi_networks is not None:
|
||||
updates.append('wifi_networks = ?')
|
||||
params.append(json.dumps(wifi_networks))
|
||||
if wifi_clients is not None:
|
||||
updates.append('wifi_clients = ?')
|
||||
params.append(json.dumps(wifi_clients))
|
||||
if wifi_networks is not None:
|
||||
updates.append('wifi_networks = ?')
|
||||
params.append(json.dumps(wifi_networks))
|
||||
if wifi_clients is not None:
|
||||
updates.append('wifi_clients = ?')
|
||||
params.append(json.dumps(wifi_clients))
|
||||
if bt_devices is not None:
|
||||
updates.append('bt_devices = ?')
|
||||
params.append(json.dumps(bt_devices))
|
||||
@@ -1267,127 +1441,127 @@ def get_all_known_devices(
|
||||
]
|
||||
|
||||
|
||||
def delete_known_device(identifier: str) -> bool:
|
||||
"""Remove a device from the known-good registry."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
'DELETE FROM tscm_known_devices WHERE identifier = ?',
|
||||
(identifier.upper(),)
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TSCM Schedule Functions
|
||||
# =============================================================================
|
||||
|
||||
def create_tscm_schedule(
|
||||
name: str,
|
||||
cron_expression: str,
|
||||
sweep_type: str = 'standard',
|
||||
baseline_id: int | None = None,
|
||||
zone_name: str | None = None,
|
||||
enabled: bool = True,
|
||||
notify_on_threat: bool = True,
|
||||
notify_email: str | None = None,
|
||||
last_run: str | None = None,
|
||||
next_run: str | None = None,
|
||||
) -> int:
|
||||
"""Create a new TSCM sweep schedule."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute('''
|
||||
INSERT INTO tscm_schedules
|
||||
(name, baseline_id, zone_name, cron_expression, sweep_type,
|
||||
enabled, last_run, next_run, notify_on_threat, notify_email)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
name,
|
||||
baseline_id,
|
||||
zone_name,
|
||||
cron_expression,
|
||||
sweep_type,
|
||||
1 if enabled else 0,
|
||||
last_run,
|
||||
next_run,
|
||||
1 if notify_on_threat else 0,
|
||||
notify_email,
|
||||
))
|
||||
return cursor.lastrowid
|
||||
|
||||
|
||||
def get_tscm_schedule(schedule_id: int) -> dict | None:
|
||||
"""Get a TSCM schedule by ID."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
'SELECT * FROM tscm_schedules WHERE id = ?',
|
||||
(schedule_id,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def get_all_tscm_schedules(
|
||||
enabled: bool | None = None,
|
||||
limit: int = 200
|
||||
) -> list[dict]:
|
||||
"""Get all TSCM schedules."""
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
if enabled is not None:
|
||||
conditions.append('enabled = ?')
|
||||
params.append(1 if enabled else 0)
|
||||
|
||||
where_clause = f'WHERE {" AND ".join(conditions)}' if conditions else ''
|
||||
params.append(limit)
|
||||
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(f'''
|
||||
SELECT * FROM tscm_schedules
|
||||
{where_clause}
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
''', params)
|
||||
return [dict(row) for row in cursor]
|
||||
|
||||
|
||||
def update_tscm_schedule(schedule_id: int, **fields) -> bool:
|
||||
"""Update a TSCM schedule."""
|
||||
if not fields:
|
||||
return False
|
||||
|
||||
updates = []
|
||||
params = []
|
||||
|
||||
for key, value in fields.items():
|
||||
updates.append(f'{key} = ?')
|
||||
params.append(value)
|
||||
|
||||
params.append(schedule_id)
|
||||
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
f'UPDATE tscm_schedules SET {", ".join(updates)} WHERE id = ?',
|
||||
params
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
def delete_tscm_schedule(schedule_id: int) -> bool:
|
||||
"""Delete a TSCM schedule."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
'DELETE FROM tscm_schedules WHERE id = ?',
|
||||
(schedule_id,)
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
def is_known_good_device(identifier: str, location: str | None = None) -> dict | None:
|
||||
"""Check if a device is in the known-good registry for a location."""
|
||||
with get_db() as conn:
|
||||
if location:
|
||||
cursor = conn.execute('''
|
||||
def delete_known_device(identifier: str) -> bool:
|
||||
"""Remove a device from the known-good registry."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
'DELETE FROM tscm_known_devices WHERE identifier = ?',
|
||||
(identifier.upper(),)
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TSCM Schedule Functions
|
||||
# =============================================================================
|
||||
|
||||
def create_tscm_schedule(
|
||||
name: str,
|
||||
cron_expression: str,
|
||||
sweep_type: str = 'standard',
|
||||
baseline_id: int | None = None,
|
||||
zone_name: str | None = None,
|
||||
enabled: bool = True,
|
||||
notify_on_threat: bool = True,
|
||||
notify_email: str | None = None,
|
||||
last_run: str | None = None,
|
||||
next_run: str | None = None,
|
||||
) -> int:
|
||||
"""Create a new TSCM sweep schedule."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute('''
|
||||
INSERT INTO tscm_schedules
|
||||
(name, baseline_id, zone_name, cron_expression, sweep_type,
|
||||
enabled, last_run, next_run, notify_on_threat, notify_email)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (
|
||||
name,
|
||||
baseline_id,
|
||||
zone_name,
|
||||
cron_expression,
|
||||
sweep_type,
|
||||
1 if enabled else 0,
|
||||
last_run,
|
||||
next_run,
|
||||
1 if notify_on_threat else 0,
|
||||
notify_email,
|
||||
))
|
||||
return cursor.lastrowid
|
||||
|
||||
|
||||
def get_tscm_schedule(schedule_id: int) -> dict | None:
|
||||
"""Get a TSCM schedule by ID."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
'SELECT * FROM tscm_schedules WHERE id = ?',
|
||||
(schedule_id,)
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def get_all_tscm_schedules(
|
||||
enabled: bool | None = None,
|
||||
limit: int = 200
|
||||
) -> list[dict]:
|
||||
"""Get all TSCM schedules."""
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
if enabled is not None:
|
||||
conditions.append('enabled = ?')
|
||||
params.append(1 if enabled else 0)
|
||||
|
||||
where_clause = f'WHERE {" AND ".join(conditions)}' if conditions else ''
|
||||
params.append(limit)
|
||||
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(f'''
|
||||
SELECT * FROM tscm_schedules
|
||||
{where_clause}
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
''', params)
|
||||
return [dict(row) for row in cursor]
|
||||
|
||||
|
||||
def update_tscm_schedule(schedule_id: int, **fields) -> bool:
|
||||
"""Update a TSCM schedule."""
|
||||
if not fields:
|
||||
return False
|
||||
|
||||
updates = []
|
||||
params = []
|
||||
|
||||
for key, value in fields.items():
|
||||
updates.append(f'{key} = ?')
|
||||
params.append(value)
|
||||
|
||||
params.append(schedule_id)
|
||||
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
f'UPDATE tscm_schedules SET {", ".join(updates)} WHERE id = ?',
|
||||
params
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
def delete_tscm_schedule(schedule_id: int) -> bool:
|
||||
"""Delete a TSCM schedule."""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute(
|
||||
'DELETE FROM tscm_schedules WHERE id = ?',
|
||||
(schedule_id,)
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
|
||||
def is_known_good_device(identifier: str, location: str | None = None) -> dict | None:
|
||||
"""Check if a device is in the known-good registry for a location."""
|
||||
with get_db() as conn:
|
||||
if location:
|
||||
cursor = conn.execute('''
|
||||
SELECT * FROM tscm_known_devices
|
||||
WHERE identifier = ? AND (location = ? OR scope = 'global')
|
||||
''', (identifier.upper(), location))
|
||||
@@ -2123,3 +2297,61 @@ def cleanup_old_payloads(max_age_hours: int = 24) -> int:
|
||||
WHERE received_at < datetime('now', ?)
|
||||
''', (f'-{max_age_hours} hours',))
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# GSM Cleanup Functions
|
||||
# =============================================================================
|
||||
|
||||
def cleanup_old_gsm_signals(max_age_days: int = 60) -> int:
|
||||
"""
|
||||
Remove old GSM signal observations (60-day archive).
|
||||
|
||||
Args:
|
||||
max_age_days: Maximum age in days (default: 60)
|
||||
|
||||
Returns:
|
||||
Number of deleted entries
|
||||
"""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute('''
|
||||
DELETE FROM gsm_signals
|
||||
WHERE timestamp < datetime('now', ?)
|
||||
''', (f'-{max_age_days} days',))
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def cleanup_old_gsm_tmsi_log(max_age_hours: int = 24) -> int:
|
||||
"""
|
||||
Remove old TMSI log entries (24-hour buffer for crowd density).
|
||||
|
||||
Args:
|
||||
max_age_hours: Maximum age in hours (default: 24)
|
||||
|
||||
Returns:
|
||||
Number of deleted entries
|
||||
"""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute('''
|
||||
DELETE FROM gsm_tmsi_log
|
||||
WHERE timestamp < datetime('now', ?)
|
||||
''', (f'-{max_age_hours} hours',))
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def cleanup_old_gsm_velocity_log(max_age_hours: int = 1) -> int:
|
||||
"""
|
||||
Remove old velocity log entries (1-hour buffer for movement tracking).
|
||||
|
||||
Args:
|
||||
max_age_hours: Maximum age in hours (default: 1)
|
||||
|
||||
Returns:
|
||||
Number of deleted entries
|
||||
"""
|
||||
with get_db() as conn:
|
||||
cursor = conn.execute('''
|
||||
DELETE FROM gsm_velocity_log
|
||||
WHERE timestamp < datetime('now', ?)
|
||||
''', (f'-{max_age_hours} hours',))
|
||||
return cursor.rowcount
|
||||
|
||||
@@ -443,6 +443,38 @@ TOOL_DEPENDENCIES = {
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
'gsm': {
|
||||
'name': 'GSM Intelligence',
|
||||
'tools': {
|
||||
'grgsm_scanner': {
|
||||
'required': True,
|
||||
'description': 'gr-gsm scanner for finding GSM towers',
|
||||
'install': {
|
||||
'apt': 'Build gr-gsm from source: https://github.com/bkerler/gr-gsm',
|
||||
'brew': 'brew install gr-gsm (may require manual build)',
|
||||
'manual': 'https://github.com/bkerler/gr-gsm'
|
||||
}
|
||||
},
|
||||
'grgsm_livemon': {
|
||||
'required': True,
|
||||
'description': 'gr-gsm live monitor for decoding GSM signals',
|
||||
'install': {
|
||||
'apt': 'Included with gr-gsm package',
|
||||
'brew': 'Included with gr-gsm',
|
||||
'manual': 'Included with gr-gsm'
|
||||
}
|
||||
},
|
||||
'tshark': {
|
||||
'required': True,
|
||||
'description': 'Wireshark CLI for parsing GSM packets',
|
||||
'install': {
|
||||
'apt': 'sudo apt-get install tshark',
|
||||
'brew': 'brew install wireshark',
|
||||
'manual': 'https://www.wireshark.org/download.html'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
"""GSM Cell Tower Geocoding Service.
|
||||
|
||||
Provides hybrid cache-first geocoding with async API fallback for cell towers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import queue
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
import config
|
||||
from utils.database import get_db
|
||||
|
||||
logger = logging.getLogger('intercept.gsm_geocoding')
|
||||
|
||||
# Queue for pending geocoding requests
|
||||
_geocoding_queue = queue.Queue(maxsize=100)
|
||||
|
||||
|
||||
def lookup_cell_coordinates(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None:
|
||||
"""
|
||||
Lookup cell tower coordinates with cache-first strategy.
|
||||
|
||||
Strategy:
|
||||
1. Check gsm_cells table (cache) - fast synchronous lookup
|
||||
2. If not found, return None (caller decides whether to use API)
|
||||
|
||||
Args:
|
||||
mcc: Mobile Country Code
|
||||
mnc: Mobile Network Code
|
||||
lac: Location Area Code
|
||||
cid: Cell ID
|
||||
|
||||
Returns:
|
||||
dict with keys: lat, lon, source='cache', azimuth (optional),
|
||||
range_meters (optional), operator (optional), radio (optional)
|
||||
Returns None if not found in cache.
|
||||
"""
|
||||
try:
|
||||
with get_db() as conn:
|
||||
result = conn.execute('''
|
||||
SELECT lat, lon, azimuth, range_meters, operator, radio
|
||||
FROM gsm_cells
|
||||
WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ?
|
||||
''', (mcc, mnc, lac, cid)).fetchone()
|
||||
|
||||
if result:
|
||||
return {
|
||||
'lat': result['lat'],
|
||||
'lon': result['lon'],
|
||||
'source': 'cache',
|
||||
'azimuth': result['azimuth'],
|
||||
'range_meters': result['range_meters'],
|
||||
'operator': result['operator'],
|
||||
'radio': result['radio']
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error looking up coordinates from cache: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None:
|
||||
"""
|
||||
Lookup cell tower from OpenCellID API and cache result.
|
||||
|
||||
Args:
|
||||
mcc: Mobile Country Code
|
||||
mnc: Mobile Network Code
|
||||
lac: Location Area Code
|
||||
cid: Cell ID
|
||||
|
||||
Returns:
|
||||
dict with keys: lat, lon, source='api', azimuth (optional),
|
||||
range_meters (optional), operator (optional), radio (optional)
|
||||
Returns None if API call fails or cell not found.
|
||||
"""
|
||||
try:
|
||||
api_url = config.GSM_OPENCELLID_API_URL
|
||||
params = {
|
||||
'key': config.GSM_OPENCELLID_API_KEY,
|
||||
'mcc': mcc,
|
||||
'mnc': mnc,
|
||||
'lac': lac,
|
||||
'cellid': cid,
|
||||
'format': 'json'
|
||||
}
|
||||
|
||||
response = requests.get(api_url, params=params, timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
cell_data = response.json()
|
||||
|
||||
# Cache the result
|
||||
with get_db() as conn:
|
||||
conn.execute('''
|
||||
INSERT OR REPLACE INTO gsm_cells
|
||||
(mcc, mnc, lac, cid, lat, lon, azimuth, range_meters, samples, radio, operator, last_verified)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
''', (
|
||||
mcc, mnc, lac, cid,
|
||||
cell_data.get('lat'),
|
||||
cell_data.get('lon'),
|
||||
cell_data.get('azimuth'),
|
||||
cell_data.get('range'),
|
||||
cell_data.get('samples'),
|
||||
cell_data.get('radio'),
|
||||
cell_data.get('operator')
|
||||
))
|
||||
conn.commit()
|
||||
|
||||
logger.info(f"Cached cell tower from API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
|
||||
|
||||
return {
|
||||
'lat': cell_data.get('lat'),
|
||||
'lon': cell_data.get('lon'),
|
||||
'source': 'api',
|
||||
'azimuth': cell_data.get('azimuth'),
|
||||
'range_meters': cell_data.get('range'),
|
||||
'operator': cell_data.get('operator'),
|
||||
'radio': cell_data.get('radio')
|
||||
}
|
||||
else:
|
||||
logger.warning(f"OpenCellID API returned {response.status_code} for MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calling OpenCellID API: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def enrich_tower_data(tower_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Enrich tower data with coordinates using cache-first strategy.
|
||||
|
||||
If coordinates found in cache, adds them immediately.
|
||||
If not found, marks as 'pending' and queues for background API lookup.
|
||||
|
||||
Args:
|
||||
tower_data: Dictionary with keys mcc, mnc, lac, cid (and other tower data)
|
||||
|
||||
Returns:
|
||||
Enriched tower_data dict with added fields:
|
||||
- lat, lon (if found in cache)
|
||||
- status='pending' (if needs API lookup)
|
||||
- source='cache' (if from cache)
|
||||
"""
|
||||
mcc = tower_data.get('mcc')
|
||||
mnc = tower_data.get('mnc')
|
||||
lac = tower_data.get('lac')
|
||||
cid = tower_data.get('cid')
|
||||
|
||||
# Validate required fields
|
||||
if not all([mcc is not None, mnc is not None, lac is not None, cid is not None]):
|
||||
logger.warning(f"Tower data missing required fields: {tower_data}")
|
||||
return tower_data
|
||||
|
||||
# Try cache lookup
|
||||
coords = lookup_cell_coordinates(mcc, mnc, lac, cid)
|
||||
|
||||
if coords:
|
||||
# Found in cache - add coordinates immediately
|
||||
tower_data['lat'] = coords['lat']
|
||||
tower_data['lon'] = coords['lon']
|
||||
tower_data['source'] = 'cache'
|
||||
|
||||
# Add optional fields if available
|
||||
if coords.get('azimuth') is not None:
|
||||
tower_data['azimuth'] = coords['azimuth']
|
||||
if coords.get('range_meters') is not None:
|
||||
tower_data['range_meters'] = coords['range_meters']
|
||||
if coords.get('operator'):
|
||||
tower_data['operator'] = coords['operator']
|
||||
if coords.get('radio'):
|
||||
tower_data['radio'] = coords['radio']
|
||||
|
||||
logger.debug(f"Cache hit for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
|
||||
else:
|
||||
# Not in cache - mark as pending and queue for API lookup
|
||||
tower_data['status'] = 'pending'
|
||||
tower_data['source'] = 'unknown'
|
||||
|
||||
# Queue for background geocoding (non-blocking)
|
||||
try:
|
||||
_geocoding_queue.put_nowait(tower_data.copy())
|
||||
logger.debug(f"Queued tower for geocoding: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
|
||||
except queue.Full:
|
||||
logger.warning("Geocoding queue full, dropping tower")
|
||||
|
||||
return tower_data
|
||||
|
||||
|
||||
def get_geocoding_queue() -> queue.Queue:
|
||||
"""Get the geocoding queue for the background worker."""
|
||||
return _geocoding_queue
|
||||
@@ -28,3 +28,4 @@ wifi_logger = get_logger('intercept.wifi')
|
||||
bluetooth_logger = get_logger('intercept.bluetooth')
|
||||
adsb_logger = get_logger('intercept.adsb')
|
||||
satellite_logger = get_logger('intercept.satellite')
|
||||
gsm_spy_logger = get_logger('intercept.gsm_spy')
|
||||
|
||||
@@ -185,6 +185,43 @@ class AirspyCommandBuilder(CommandBuilder):
|
||||
|
||||
return cmd
|
||||
|
||||
def build_iq_capture_command(
|
||||
self,
|
||||
device: SDRDevice,
|
||||
frequency_mhz: float,
|
||||
sample_rate: int = 2048000,
|
||||
gain: Optional[float] = None,
|
||||
ppm: Optional[int] = None,
|
||||
bias_t: bool = False,
|
||||
output_format: str = 'cu8',
|
||||
) -> list[str]:
|
||||
"""
|
||||
Build rx_sdr command for raw I/Q capture with Airspy.
|
||||
|
||||
Outputs unsigned 8-bit I/Q pairs to stdout for waterfall display.
|
||||
"""
|
||||
device_str = self._build_device_string(device)
|
||||
freq_hz = int(frequency_mhz * 1e6)
|
||||
|
||||
cmd = [
|
||||
'rx_sdr',
|
||||
'-d', device_str,
|
||||
'-f', str(freq_hz),
|
||||
'-s', str(sample_rate),
|
||||
'-F', 'CU8',
|
||||
]
|
||||
|
||||
if gain is not None and gain > 0:
|
||||
cmd.extend(['-g', self._format_gain(gain)])
|
||||
|
||||
if bias_t:
|
||||
cmd.append('-T')
|
||||
|
||||
# Output to stdout
|
||||
cmd.append('-')
|
||||
|
||||
return cmd
|
||||
|
||||
def get_capabilities(self) -> SDRCapabilities:
|
||||
"""Return Airspy capabilities."""
|
||||
return self.CAPABILITIES
|
||||
|
||||
@@ -186,6 +186,41 @@ class CommandBuilder(ABC):
|
||||
"""Return hardware capabilities for this SDR type."""
|
||||
pass
|
||||
|
||||
def build_iq_capture_command(
|
||||
self,
|
||||
device: SDRDevice,
|
||||
frequency_mhz: float,
|
||||
sample_rate: int = 2048000,
|
||||
gain: Optional[float] = None,
|
||||
ppm: Optional[int] = None,
|
||||
bias_t: bool = False,
|
||||
output_format: str = 'cu8',
|
||||
) -> list[str]:
|
||||
"""
|
||||
Build raw I/Q capture command for streaming samples to stdout.
|
||||
|
||||
Used for real-time waterfall/spectrum display. Output is unsigned
|
||||
8-bit I/Q pairs (cu8) written continuously to stdout.
|
||||
|
||||
Args:
|
||||
device: The SDR device to use
|
||||
frequency_mhz: Center frequency in MHz
|
||||
sample_rate: Sample rate in Hz (default 2048000)
|
||||
gain: Gain in dB (None for auto)
|
||||
ppm: PPM frequency correction
|
||||
bias_t: Enable bias-T power (for active antennas)
|
||||
output_format: Output sample format (default 'cu8')
|
||||
|
||||
Returns:
|
||||
Command as list of strings for subprocess
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the SDR type does not support I/Q capture.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"{self.__class__.__name__} does not support raw I/Q capture"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_sdr_type(cls) -> SDRType:
|
||||
|
||||
@@ -185,6 +185,44 @@ class HackRFCommandBuilder(CommandBuilder):
|
||||
|
||||
return cmd
|
||||
|
||||
def build_iq_capture_command(
|
||||
self,
|
||||
device: SDRDevice,
|
||||
frequency_mhz: float,
|
||||
sample_rate: int = 2048000,
|
||||
gain: Optional[float] = None,
|
||||
ppm: Optional[int] = None,
|
||||
bias_t: bool = False,
|
||||
output_format: str = 'cu8',
|
||||
) -> list[str]:
|
||||
"""
|
||||
Build rx_sdr command for raw I/Q capture with HackRF.
|
||||
|
||||
Outputs unsigned 8-bit I/Q pairs to stdout for waterfall display.
|
||||
"""
|
||||
device_str = self._build_device_string(device)
|
||||
freq_hz = int(frequency_mhz * 1e6)
|
||||
|
||||
cmd = [
|
||||
'rx_sdr',
|
||||
'-d', device_str,
|
||||
'-f', str(freq_hz),
|
||||
'-s', str(sample_rate),
|
||||
'-F', 'CU8',
|
||||
]
|
||||
|
||||
if gain is not None and gain > 0:
|
||||
lna, vga = self._split_gain(gain)
|
||||
cmd.extend(['-g', f'LNA={lna},VGA={vga}'])
|
||||
|
||||
if bias_t:
|
||||
cmd.append('-T')
|
||||
|
||||
# Output to stdout
|
||||
cmd.append('-')
|
||||
|
||||
return cmd
|
||||
|
||||
def get_capabilities(self) -> SDRCapabilities:
|
||||
"""Return HackRF capabilities."""
|
||||
return self.CAPABILITIES
|
||||
|
||||
@@ -162,6 +162,41 @@ class LimeSDRCommandBuilder(CommandBuilder):
|
||||
|
||||
return cmd
|
||||
|
||||
def build_iq_capture_command(
|
||||
self,
|
||||
device: SDRDevice,
|
||||
frequency_mhz: float,
|
||||
sample_rate: int = 2048000,
|
||||
gain: Optional[float] = None,
|
||||
ppm: Optional[int] = None,
|
||||
bias_t: bool = False,
|
||||
output_format: str = 'cu8',
|
||||
) -> list[str]:
|
||||
"""
|
||||
Build rx_sdr command for raw I/Q capture with LimeSDR.
|
||||
|
||||
Outputs unsigned 8-bit I/Q pairs to stdout for waterfall display.
|
||||
Note: LimeSDR does not support bias-T, parameter is ignored.
|
||||
"""
|
||||
device_str = self._build_device_string(device)
|
||||
freq_hz = int(frequency_mhz * 1e6)
|
||||
|
||||
cmd = [
|
||||
'rx_sdr',
|
||||
'-d', device_str,
|
||||
'-f', str(freq_hz),
|
||||
'-s', str(sample_rate),
|
||||
'-F', 'CU8',
|
||||
]
|
||||
|
||||
if gain is not None and gain > 0:
|
||||
cmd.extend(['-g', f'LNAH={int(gain)}'])
|
||||
|
||||
# Output to stdout
|
||||
cmd.append('-')
|
||||
|
||||
return cmd
|
||||
|
||||
def get_capabilities(self) -> SDRCapabilities:
|
||||
"""Return LimeSDR capabilities."""
|
||||
return self.CAPABILITIES
|
||||
|
||||
@@ -231,6 +231,45 @@ class RTLSDRCommandBuilder(CommandBuilder):
|
||||
|
||||
return cmd
|
||||
|
||||
def build_iq_capture_command(
|
||||
self,
|
||||
device: SDRDevice,
|
||||
frequency_mhz: float,
|
||||
sample_rate: int = 2048000,
|
||||
gain: Optional[float] = None,
|
||||
ppm: Optional[int] = None,
|
||||
bias_t: bool = False,
|
||||
output_format: str = 'cu8',
|
||||
) -> list[str]:
|
||||
"""
|
||||
Build rtl_sdr command for raw I/Q capture.
|
||||
|
||||
Outputs unsigned 8-bit I/Q pairs to stdout for waterfall display.
|
||||
"""
|
||||
rtl_sdr_path = get_tool_path('rtl_sdr') or 'rtl_sdr'
|
||||
freq_hz = int(frequency_mhz * 1e6)
|
||||
|
||||
cmd = [
|
||||
rtl_sdr_path,
|
||||
'-d', self._get_device_arg(device),
|
||||
'-f', str(freq_hz),
|
||||
'-s', str(sample_rate),
|
||||
]
|
||||
|
||||
if gain is not None and gain > 0:
|
||||
cmd.extend(['-g', str(gain)])
|
||||
|
||||
if ppm is not None and ppm != 0:
|
||||
cmd.extend(['-p', str(ppm)])
|
||||
|
||||
if bias_t:
|
||||
cmd.append('-T')
|
||||
|
||||
# Output to stdout
|
||||
cmd.append('-')
|
||||
|
||||
return cmd
|
||||
|
||||
def get_capabilities(self) -> SDRCapabilities:
|
||||
"""Return RTL-SDR capabilities."""
|
||||
return self.CAPABILITIES
|
||||
|
||||
@@ -163,6 +163,43 @@ class SDRPlayCommandBuilder(CommandBuilder):
|
||||
|
||||
return cmd
|
||||
|
||||
def build_iq_capture_command(
|
||||
self,
|
||||
device: SDRDevice,
|
||||
frequency_mhz: float,
|
||||
sample_rate: int = 2048000,
|
||||
gain: Optional[float] = None,
|
||||
ppm: Optional[int] = None,
|
||||
bias_t: bool = False,
|
||||
output_format: str = 'cu8',
|
||||
) -> list[str]:
|
||||
"""
|
||||
Build rx_sdr command for raw I/Q capture with SDRPlay.
|
||||
|
||||
Outputs unsigned 8-bit I/Q pairs to stdout for waterfall display.
|
||||
"""
|
||||
device_str = self._build_device_string(device)
|
||||
freq_hz = int(frequency_mhz * 1e6)
|
||||
|
||||
cmd = [
|
||||
'rx_sdr',
|
||||
'-d', device_str,
|
||||
'-f', str(freq_hz),
|
||||
'-s', str(sample_rate),
|
||||
'-F', 'CU8',
|
||||
]
|
||||
|
||||
if gain is not None and gain > 0:
|
||||
cmd.extend(['-g', f'IFGR={int(gain)}'])
|
||||
|
||||
if bias_t:
|
||||
cmd.append('-T')
|
||||
|
||||
# Output to stdout
|
||||
cmd.append('-')
|
||||
|
||||
return cmd
|
||||
|
||||
def get_capabilities(self) -> SDRCapabilities:
|
||||
"""Return SDRPlay capabilities."""
|
||||
return self.CAPABILITIES
|
||||
|
||||
@@ -225,7 +225,7 @@ class SSTVDecoder:
|
||||
self._rtl_process = None
|
||||
self._running = False
|
||||
self._lock = threading.Lock()
|
||||
self._callback: Callable[[DecodeProgress], None] | None = None
|
||||
self._callback: Callable[[dict], None] | None = None
|
||||
self._output_dir = Path(output_dir) if output_dir else Path('instance/sstv_images')
|
||||
self._url_prefix = url_prefix
|
||||
self._images: list[SSTVImage] = []
|
||||
@@ -253,7 +253,7 @@ class SSTVDecoder:
|
||||
"""Return name of available decoder. Always available with pure Python."""
|
||||
return 'python-sstv'
|
||||
|
||||
def set_callback(self, callback: Callable[[DecodeProgress], None]) -> None:
|
||||
def set_callback(self, callback: Callable[[dict], None]) -> None:
|
||||
"""Set callback for decode progress updates."""
|
||||
self._callback = callback
|
||||
|
||||
@@ -420,6 +420,10 @@ class SSTVDecoder:
|
||||
|
||||
chunk_counter += 1
|
||||
|
||||
# Scope: compute RMS/peak from raw int16 samples every chunk
|
||||
rms_val = int(np.sqrt(np.mean(raw_samples.astype(np.float64) ** 2)))
|
||||
peak_val = int(np.max(np.abs(raw_samples)))
|
||||
|
||||
if image_decoder is not None:
|
||||
# Currently decoding an image
|
||||
complete = image_decoder.feed(samples)
|
||||
@@ -447,6 +451,7 @@ class SSTVDecoder:
|
||||
message=f'Decoding {current_mode_name}: {pct}%',
|
||||
partial_image=partial_url,
|
||||
))
|
||||
self._emit_scope(rms_val, peak_val, 'decoding')
|
||||
|
||||
if complete:
|
||||
# Save image
|
||||
@@ -479,6 +484,7 @@ class SSTVDecoder:
|
||||
vis_detector.reset()
|
||||
|
||||
# Emit signal level metrics every ~500ms (every 5th 100ms chunk)
|
||||
scope_tone: str | None = None
|
||||
if chunk_counter % 5 == 0 and image_decoder is None:
|
||||
rms = float(np.sqrt(np.mean(samples ** 2)))
|
||||
signal_level = min(100, int(rms * 500))
|
||||
@@ -501,6 +507,8 @@ class SSTVDecoder:
|
||||
else:
|
||||
sstv_tone = None
|
||||
|
||||
scope_tone = sstv_tone
|
||||
|
||||
self._emit_progress(DecodeProgress(
|
||||
status='detecting',
|
||||
message='Listening...',
|
||||
@@ -509,6 +517,8 @@ class SSTVDecoder:
|
||||
vis_state=vis_detector.state.value,
|
||||
))
|
||||
|
||||
self._emit_scope(rms_val, peak_val, scope_tone)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in decode thread: {e}")
|
||||
if not self._running:
|
||||
@@ -736,10 +746,18 @@ class SSTVDecoder:
|
||||
"""Emit progress update to callback."""
|
||||
if self._callback:
|
||||
try:
|
||||
self._callback(progress)
|
||||
self._callback(progress.to_dict())
|
||||
except Exception as e:
|
||||
logger.error(f"Error in progress callback: {e}")
|
||||
|
||||
def _emit_scope(self, rms: int, peak: int, tone: str | None = None) -> None:
|
||||
"""Emit scope signal levels to callback."""
|
||||
if self._callback:
|
||||
try:
|
||||
self._callback({'type': 'sstv_scope', 'rms': rms, 'peak': peak, 'tone': tone})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def decode_file(self, audio_path: str | Path) -> list[SSTVImage]:
|
||||
"""Decode SSTV image(s) from an audio file.
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
"""FFT pipeline for real-time waterfall display.
|
||||
|
||||
Converts raw I/Q samples from SDR hardware into quantized power spectrum
|
||||
frames suitable for binary WebSocket transmission.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def cu8_to_complex(raw: bytes) -> np.ndarray:
|
||||
"""Convert unsigned 8-bit I/Q bytes to complex64.
|
||||
|
||||
RTL-SDR (and rx_sdr with -F cu8) outputs interleaved unsigned 8-bit
|
||||
I/Q pairs where 128 is the zero point.
|
||||
|
||||
Args:
|
||||
raw: Raw bytes, length must be even (I/Q pairs).
|
||||
|
||||
Returns:
|
||||
Complex64 array of length len(raw) // 2.
|
||||
"""
|
||||
iq = np.frombuffer(raw, dtype=np.uint8).astype(np.float32)
|
||||
# Normalize: 0 -> -1.0, 128 -> ~0.0, 255 -> +1.0
|
||||
iq = (iq - 127.5) / 127.5
|
||||
return iq[0::2] + 1j * iq[1::2]
|
||||
|
||||
|
||||
def compute_power_spectrum(
|
||||
samples: np.ndarray,
|
||||
fft_size: int = 1024,
|
||||
avg_count: int = 4,
|
||||
) -> np.ndarray:
|
||||
"""Compute averaged power spectrum in dBm.
|
||||
|
||||
Applies a Hann window, computes FFT, converts to power (dB),
|
||||
and averages over multiple segments.
|
||||
|
||||
Args:
|
||||
samples: Complex64 array, length >= fft_size * avg_count.
|
||||
fft_size: Number of FFT bins.
|
||||
avg_count: Number of segments to average.
|
||||
|
||||
Returns:
|
||||
Float32 array of length fft_size with power in dB (fftshift'd).
|
||||
"""
|
||||
window = np.hanning(fft_size).astype(np.float32)
|
||||
accum = np.zeros(fft_size, dtype=np.float32)
|
||||
actual_avg = 0
|
||||
|
||||
for i in range(avg_count):
|
||||
offset = i * fft_size
|
||||
if offset + fft_size > len(samples):
|
||||
break
|
||||
segment = samples[offset : offset + fft_size] * window
|
||||
spectrum = np.fft.fft(segment)
|
||||
power = np.real(spectrum * np.conj(spectrum))
|
||||
# Avoid log10(0)
|
||||
power = np.maximum(power, 1e-20)
|
||||
accum += 10.0 * np.log10(power)
|
||||
actual_avg += 1
|
||||
|
||||
if actual_avg == 0:
|
||||
return np.full(fft_size, -100.0, dtype=np.float32)
|
||||
|
||||
accum /= actual_avg
|
||||
return np.fft.fftshift(accum).astype(np.float32)
|
||||
|
||||
|
||||
def quantize_to_uint8(
|
||||
power_db: np.ndarray,
|
||||
db_min: float | None = None,
|
||||
db_max: float | None = None,
|
||||
) -> bytes:
|
||||
"""Clamp and scale dB values to 0-255.
|
||||
|
||||
When *db_min* / *db_max* are ``None`` (the default) the range is
|
||||
derived from the data so the full colour palette is always used.
|
||||
|
||||
Args:
|
||||
power_db: Float32 array of power values in dB.
|
||||
db_min: Value mapped to 0 (auto if None).
|
||||
db_max: Value mapped to 255 (auto if None).
|
||||
|
||||
Returns:
|
||||
Bytes of length len(power_db), each in [0, 255].
|
||||
"""
|
||||
if db_min is None or db_max is None:
|
||||
actual_min = float(np.min(power_db))
|
||||
actual_max = float(np.max(power_db))
|
||||
# Guarantee at least 1 dB of dynamic range
|
||||
if actual_max - actual_min < 1.0:
|
||||
actual_max = actual_min + 1.0
|
||||
if db_min is None:
|
||||
db_min = actual_min
|
||||
if db_max is None:
|
||||
db_max = actual_max
|
||||
|
||||
db_range = db_max - db_min
|
||||
if db_range <= 0:
|
||||
db_range = 1.0
|
||||
scaled = (power_db - db_min) / db_range * 255.0
|
||||
clamped = np.clip(scaled, 0, 255).astype(np.uint8)
|
||||
return clamped.tobytes()
|
||||
|
||||
|
||||
def build_binary_frame(
|
||||
start_freq: float,
|
||||
end_freq: float,
|
||||
quantized_bins: bytes,
|
||||
) -> bytes:
|
||||
"""Pack a binary waterfall frame for WebSocket transmission.
|
||||
|
||||
Wire format (little-endian):
|
||||
[uint8 msg_type=0x01]
|
||||
[float32 start_freq]
|
||||
[float32 end_freq]
|
||||
[uint16 bin_count]
|
||||
[uint8[] bins]
|
||||
|
||||
Total size = 11 + bin_count bytes.
|
||||
|
||||
Args:
|
||||
start_freq: Start frequency in MHz.
|
||||
end_freq: End frequency in MHz.
|
||||
quantized_bins: Pre-quantized uint8 bin data.
|
||||
|
||||
Returns:
|
||||
Binary frame bytes.
|
||||
"""
|
||||
bin_count = len(quantized_bins)
|
||||
header = struct.pack('<BffH', 0x01, start_freq, end_freq, bin_count)
|
||||
return header + quantized_bins
|
||||
Reference in New Issue
Block a user