Merge branch 'pr-124'

# Conflicts:
#	app.py
#	routes/__init__.py
#	utils/database.py

utils/cleanup.py

@@ -142,7 +142,7 @@ class DataStore:
 
 
 class CleanupManager:
-    """Manages periodic cleanup of multiple data stores."""
+    """Manages periodic cleanup of multiple data stores and database tables."""
 
     def __init__(self, interval: float = 60.0):
         """
@@ -152,9 +152,11 @@ class CleanupManager:
             interval: Cleanup interval in seconds
         """
         self.stores: list[DataStore] = []
+        self.db_cleanup_funcs: list[tuple[callable, int]] = []  # (func, interval_multiplier)
         self.interval = interval
         self._timer: threading.Timer | None = None
         self._running = False
+        self._cleanup_count = 0
         self._lock = threading.Lock()
 
     def register(self, store: DataStore) -> None:
@@ -169,6 +171,17 @@ class CleanupManager:
             if store in self.stores:
                 self.stores.remove(store)
 
+    def register_db_cleanup(self, func: callable, interval_multiplier: int = 60) -> None:
+        """
+        Register a database cleanup function.
+
+        Args:
+            func: Cleanup function to call (should return number of deleted rows)
+            interval_multiplier: How many cleanup cycles to wait between calls (default: 60 = 1 hour if interval is 60s)
+        """
+        with self._lock:
+            self.db_cleanup_funcs.append((func, interval_multiplier))
+
     def start(self) -> None:
         """Start the cleanup timer."""
         with self._lock:
@@ -194,11 +207,15 @@ class CleanupManager:
         self._timer.start()
 
     def _run_cleanup(self) -> None:
-        """Run cleanup on all registered stores."""
+        """Run cleanup on all registered stores and database tables."""
         total_cleaned = 0
 
+        # Cleanup in-memory data stores
         with self._lock:
             stores = list(self.stores)
+            db_funcs = list(self.db_cleanup_funcs)
+            self._cleanup_count += 1
+            current_count = self._cleanup_count
 
         for store in stores:
             try:
@@ -206,6 +223,17 @@ class CleanupManager:
             except Exception as e:
                 logger.error(f"Error cleaning up {store.name}: {e}")
 
+        # Cleanup database tables (less frequently)
+        for func, interval_multiplier in db_funcs:
+            if current_count % interval_multiplier == 0:
+                try:
+                    deleted = func()
+                    if deleted > 0:
+                        logger.info(f"Database cleanup: {func.__name__} removed {deleted} rows")
+                        total_cleaned += deleted
+                except Exception as e:
+                    logger.error(f"Error in database cleanup {func.__name__}: {e}")
+
         if total_cleaned > 0:
             logger.info(f"Cleanup complete: removed {total_cleaned} stale entries")
 

utils/constants.py

@@ -274,3 +274,14 @@ MAX_DEAUTH_ALERTS_AGE_SECONDS = 300  # 5 minutes
 
 # Deauth detector sniff timeout (seconds)
 DEAUTH_SNIFF_TIMEOUT = 0.5
+
+
+# =============================================================================
+# GSM SPY (Cellular Intelligence)
+# =============================================================================
+
+# Maximum age for GSM tower/device data in DataStore (seconds)
+MAX_GSM_AGE_SECONDS = 300  # 5 minutes
+
+# Timing Advance conversion to meters
+GSM_TA_METERS_PER_UNIT = 554

utils/database.py

@@ -88,65 +88,111 @@ def init_db() -> None:
             ON signal_history(mode, device_id, timestamp)
         ''')
 
-        # Device correlation table
-        conn.execute('''
-            CREATE TABLE IF NOT EXISTS device_correlations (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                wifi_mac TEXT,
-                bt_mac TEXT,
-                confidence REAL,
-                first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                metadata TEXT,
-                UNIQUE(wifi_mac, bt_mac)
-            )
-        ''')
-
-        # Alert rules
-        conn.execute('''
-            CREATE TABLE IF NOT EXISTS alert_rules (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                name TEXT NOT NULL,
-                mode TEXT,
-                event_type TEXT,
-                match TEXT,
-                severity TEXT DEFAULT 'medium',
-                enabled BOOLEAN DEFAULT 1,
-                notify TEXT,
-                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-            )
-        ''')
-
-        # Alert events
-        conn.execute('''
-            CREATE TABLE IF NOT EXISTS alert_events (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                rule_id INTEGER,
-                mode TEXT,
-                event_type TEXT,
-                severity TEXT DEFAULT 'medium',
-                title TEXT,
-                message TEXT,
-                payload TEXT,
-                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
-            )
-        ''')
-
-        # Session recordings
-        conn.execute('''
-            CREATE TABLE IF NOT EXISTS recording_sessions (
-                id TEXT PRIMARY KEY,
-                mode TEXT NOT NULL,
-                label TEXT,
-                started_at TIMESTAMP NOT NULL,
-                stopped_at TIMESTAMP,
-                file_path TEXT NOT NULL,
-                event_count INTEGER DEFAULT 0,
-                size_bytes INTEGER DEFAULT 0,
-                metadata TEXT
-            )
-        ''')
+        # Device correlation table
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS device_correlations (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                wifi_mac TEXT,
+                bt_mac TEXT,
+                confidence REAL,
+                first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                metadata TEXT,
+                UNIQUE(wifi_mac, bt_mac)
+            )
+        ''')
+
+        # Alert rules
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS alert_rules (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                name TEXT NOT NULL,
+                mode TEXT,
+                event_type TEXT,
+                match TEXT,
+                severity TEXT DEFAULT 'medium',
+                enabled BOOLEAN DEFAULT 1,
+                notify TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        ''')
+
+        # Alert events
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS alert_events (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                rule_id INTEGER,
+                mode TEXT,
+                event_type TEXT,
+                severity TEXT DEFAULT 'medium',
+                title TEXT,
+                message TEXT,
+                payload TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
+            )
+        ''')
+
+        # Session recordings
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS recording_sessions (
+                id TEXT PRIMARY KEY,
+                mode TEXT NOT NULL,
+                label TEXT,
+                started_at TIMESTAMP NOT NULL,
+                stopped_at TIMESTAMP,
+                file_path TEXT NOT NULL,
+                event_count INTEGER DEFAULT 0,
+                size_bytes INTEGER DEFAULT 0,
+                metadata TEXT
+            )
+        ''')
+
+        # Alert rules
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS alert_rules (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                name TEXT NOT NULL,
+                mode TEXT,
+                event_type TEXT,
+                match TEXT,
+                severity TEXT DEFAULT 'medium',
+                enabled BOOLEAN DEFAULT 1,
+                notify TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        ''')
+
+        # Alert events
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS alert_events (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                rule_id INTEGER,
+                mode TEXT,
+                event_type TEXT,
+                severity TEXT DEFAULT 'medium',
+                title TEXT,
+                message TEXT,
+                payload TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
+            )
+        ''')
+
+        # Session recordings
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS recording_sessions (
+                id TEXT PRIMARY KEY,
+                mode TEXT NOT NULL,
+                label TEXT,
+                started_at TIMESTAMP NOT NULL,
+                stopped_at TIMESTAMP,
+                file_path TEXT NOT NULL,
+                event_count INTEGER DEFAULT 0,
+                size_bytes INTEGER DEFAULT 0,
+                metadata TEXT
+            )
+        ''')
 
         # Users table for authentication
         conn.execute('''
@@ -177,29 +223,29 @@ def init_db() -> None:
         # =====================================================================
 
         # TSCM Baselines - Environment snapshots for comparison
-        conn.execute('''
-            CREATE TABLE IF NOT EXISTS tscm_baselines (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                name TEXT NOT NULL,
-                location TEXT,
-                description TEXT,
-                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                wifi_networks TEXT,
-                wifi_clients TEXT,
-                bt_devices TEXT,
-                rf_frequencies TEXT,
-                gps_coords TEXT,
-                is_active BOOLEAN DEFAULT 0
-            )
-        ''')
-
-        # Ensure new columns exist for older databases
-        try:
-            columns = {row['name'] for row in conn.execute("PRAGMA table_info(tscm_baselines)")}
-            if 'wifi_clients' not in columns:
-                conn.execute('ALTER TABLE tscm_baselines ADD COLUMN wifi_clients TEXT')
-        except Exception as e:
-            logger.debug(f"Schema update skipped for tscm_baselines: {e}")
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS tscm_baselines (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                name TEXT NOT NULL,
+                location TEXT,
+                description TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                wifi_networks TEXT,
+                wifi_clients TEXT,
+                bt_devices TEXT,
+                rf_frequencies TEXT,
+                gps_coords TEXT,
+                is_active BOOLEAN DEFAULT 0
+            )
+        ''')
+
+        # Ensure new columns exist for older databases
+        try:
+            columns = {row['name'] for row in conn.execute("PRAGMA table_info(tscm_baselines)")}
+            if 'wifi_clients' not in columns:
+                conn.execute('ALTER TABLE tscm_baselines ADD COLUMN wifi_clients TEXT')
+        except Exception as e:
+            logger.debug(f"Schema update skipped for tscm_baselines: {e}")
 
         # TSCM Sweeps - Individual sweep sessions
         conn.execute('''
@@ -407,6 +453,134 @@ def init_db() -> None:
             ON tscm_cases(status, created_at)
         ''')
 
+        # =====================================================================
+        # GSM (Global System for Mobile) Intelligence Tables
+        # =====================================================================
+
+        # gsm_cells - Known cell towers (OpenCellID cache)
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS gsm_cells (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                mcc INTEGER NOT NULL,
+                mnc INTEGER NOT NULL,
+                lac INTEGER NOT NULL,
+                cid INTEGER NOT NULL,
+                lat REAL,
+                lon REAL,
+                azimuth INTEGER,
+                range_meters INTEGER,
+                samples INTEGER,
+                radio TEXT,
+                operator TEXT,
+                first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                last_verified TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                metadata TEXT,
+                UNIQUE(mcc, mnc, lac, cid)
+            )
+        ''')
+
+        # gsm_rogues - Detected rogue towers / IMSI catchers
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS gsm_rogues (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                arfcn INTEGER NOT NULL,
+                mcc INTEGER,
+                mnc INTEGER,
+                lac INTEGER,
+                cid INTEGER,
+                signal_strength REAL,
+                reason TEXT NOT NULL,
+                threat_level TEXT DEFAULT 'medium',
+                detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                location_lat REAL,
+                location_lon REAL,
+                acknowledged BOOLEAN DEFAULT 0,
+                notes TEXT,
+                metadata TEXT
+            )
+        ''')
+
+        # gsm_signals - 60-day archive of signal observations
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS gsm_signals (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                imsi TEXT,
+                tmsi TEXT,
+                mcc INTEGER,
+                mnc INTEGER,
+                lac INTEGER,
+                cid INTEGER,
+                ta_value INTEGER,
+                signal_strength REAL,
+                arfcn INTEGER,
+                timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                metadata TEXT
+            )
+        ''')
+
+        # gsm_tmsi_log - 24-hour raw pings for crowd density
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS gsm_tmsi_log (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                tmsi TEXT NOT NULL,
+                lac INTEGER,
+                cid INTEGER,
+                ta_value INTEGER,
+                timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        ''')
+
+        # gsm_velocity_log - 1-hour buffer for movement tracking
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS gsm_velocity_log (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                device_id TEXT NOT NULL,
+                prev_ta INTEGER,
+                curr_ta INTEGER,
+                prev_cid INTEGER,
+                curr_cid INTEGER,
+                timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                estimated_velocity REAL,
+                metadata TEXT
+            )
+        ''')
+
+        # GSM indexes for performance
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_gsm_cells_location
+            ON gsm_cells(lat, lon)
+        ''')
+
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_gsm_cells_identity
+            ON gsm_cells(mcc, mnc, lac, cid)
+        ''')
+
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_gsm_rogues_severity
+            ON gsm_rogues(threat_level, detected_at)
+        ''')
+
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_gsm_signals_cell_time
+            ON gsm_signals(cid, lac, timestamp)
+        ''')
+
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_gsm_signals_device
+            ON gsm_signals(imsi, tmsi, timestamp)
+        ''')
+
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_gsm_tmsi_log_time
+            ON gsm_tmsi_log(timestamp)
+        ''')
+
+        conn.execute('''
+            CREATE INDEX IF NOT EXISTS idx_gsm_velocity_log_device
+            ON gsm_velocity_log(device_id, timestamp)
+        ''')
+
         # =====================================================================
         # DSC (Digital Selective Calling) Tables
         # =====================================================================
@@ -740,16 +914,16 @@ def get_correlations(min_confidence: float = 0.5) -> list[dict]:
 # TSCM Functions
 # =============================================================================
 
-def create_tscm_baseline(
-    name: str,
-    location: str | None = None,
-    description: str | None = None,
-    wifi_networks: list | None = None,
-    wifi_clients: list | None = None,
-    bt_devices: list | None = None,
-    rf_frequencies: list | None = None,
-    gps_coords: dict | None = None
-) -> int:
+def create_tscm_baseline(
+    name: str,
+    location: str | None = None,
+    description: str | None = None,
+    wifi_networks: list | None = None,
+    wifi_clients: list | None = None,
+    bt_devices: list | None = None,
+    rf_frequencies: list | None = None,
+    gps_coords: dict | None = None
+) -> int:
     """
     Create a new TSCM baseline.
 
@@ -757,20 +931,20 @@ def create_tscm_baseline(
         The ID of the created baseline
     """
     with get_db() as conn:
-        cursor = conn.execute('''
-            INSERT INTO tscm_baselines
-            (name, location, description, wifi_networks, wifi_clients, bt_devices, rf_frequencies, gps_coords)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-        ''', (
-            name,
-            location,
-            description,
-            json.dumps(wifi_networks) if wifi_networks else None,
-            json.dumps(wifi_clients) if wifi_clients else None,
-            json.dumps(bt_devices) if bt_devices else None,
-            json.dumps(rf_frequencies) if rf_frequencies else None,
-            json.dumps(gps_coords) if gps_coords else None
-        ))
+        cursor = conn.execute('''
+            INSERT INTO tscm_baselines
+            (name, location, description, wifi_networks, wifi_clients, bt_devices, rf_frequencies, gps_coords)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        ''', (
+            name,
+            location,
+            description,
+            json.dumps(wifi_networks) if wifi_networks else None,
+            json.dumps(wifi_clients) if wifi_clients else None,
+            json.dumps(bt_devices) if bt_devices else None,
+            json.dumps(rf_frequencies) if rf_frequencies else None,
+            json.dumps(gps_coords) if gps_coords else None
+        ))
         return cursor.lastrowid
 
 
@@ -785,19 +959,19 @@ def get_tscm_baseline(baseline_id: int) -> dict | None:
         if row is None:
             return None
 
-        return {
-            'id': row['id'],
-            'name': row['name'],
-            'location': row['location'],
-            'description': row['description'],
-            'created_at': row['created_at'],
-            'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
-            'wifi_clients': json.loads(row['wifi_clients']) if row['wifi_clients'] else [],
-            'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
-            'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
-            'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
-            'is_active': bool(row['is_active'])
-        }
+        return {
+            'id': row['id'],
+            'name': row['name'],
+            'location': row['location'],
+            'description': row['description'],
+            'created_at': row['created_at'],
+            'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
+            'wifi_clients': json.loads(row['wifi_clients']) if row['wifi_clients'] else [],
+            'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
+            'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
+            'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
+            'is_active': bool(row['is_active'])
+        }
 
 
 def get_all_tscm_baselines() -> list[dict]:
@@ -839,23 +1013,23 @@ def set_active_tscm_baseline(baseline_id: int) -> bool:
         return cursor.rowcount > 0
 
 
-def update_tscm_baseline(
-    baseline_id: int,
-    wifi_networks: list | None = None,
-    wifi_clients: list | None = None,
-    bt_devices: list | None = None,
-    rf_frequencies: list | None = None
-) -> bool:
+def update_tscm_baseline(
+    baseline_id: int,
+    wifi_networks: list | None = None,
+    wifi_clients: list | None = None,
+    bt_devices: list | None = None,
+    rf_frequencies: list | None = None
+) -> bool:
     """Update baseline device lists."""
     updates = []
     params = []
 
-    if wifi_networks is not None:
-        updates.append('wifi_networks = ?')
-        params.append(json.dumps(wifi_networks))
-    if wifi_clients is not None:
-        updates.append('wifi_clients = ?')
-        params.append(json.dumps(wifi_clients))
+    if wifi_networks is not None:
+        updates.append('wifi_networks = ?')
+        params.append(json.dumps(wifi_networks))
+    if wifi_clients is not None:
+        updates.append('wifi_clients = ?')
+        params.append(json.dumps(wifi_clients))
     if bt_devices is not None:
         updates.append('bt_devices = ?')
         params.append(json.dumps(bt_devices))
@@ -1267,127 +1441,127 @@ def get_all_known_devices(
         ]
 
 
-def delete_known_device(identifier: str) -> bool:
-    """Remove a device from the known-good registry."""
-    with get_db() as conn:
-        cursor = conn.execute(
-            'DELETE FROM tscm_known_devices WHERE identifier = ?',
-            (identifier.upper(),)
-        )
-        return cursor.rowcount > 0
-
-
-# =============================================================================
-# TSCM Schedule Functions
-# =============================================================================
-
-def create_tscm_schedule(
-    name: str,
-    cron_expression: str,
-    sweep_type: str = 'standard',
-    baseline_id: int | None = None,
-    zone_name: str | None = None,
-    enabled: bool = True,
-    notify_on_threat: bool = True,
-    notify_email: str | None = None,
-    last_run: str | None = None,
-    next_run: str | None = None,
-) -> int:
-    """Create a new TSCM sweep schedule."""
-    with get_db() as conn:
-        cursor = conn.execute('''
-            INSERT INTO tscm_schedules
-            (name, baseline_id, zone_name, cron_expression, sweep_type,
-             enabled, last_run, next_run, notify_on_threat, notify_email)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        ''', (
-            name,
-            baseline_id,
-            zone_name,
-            cron_expression,
-            sweep_type,
-            1 if enabled else 0,
-            last_run,
-            next_run,
-            1 if notify_on_threat else 0,
-            notify_email,
-        ))
-        return cursor.lastrowid
-
-
-def get_tscm_schedule(schedule_id: int) -> dict | None:
-    """Get a TSCM schedule by ID."""
-    with get_db() as conn:
-        cursor = conn.execute(
-            'SELECT * FROM tscm_schedules WHERE id = ?',
-            (schedule_id,)
-        )
-        row = cursor.fetchone()
-        return dict(row) if row else None
-
-
-def get_all_tscm_schedules(
-    enabled: bool | None = None,
-    limit: int = 200
-) -> list[dict]:
-    """Get all TSCM schedules."""
-    conditions = []
-    params = []
-
-    if enabled is not None:
-        conditions.append('enabled = ?')
-        params.append(1 if enabled else 0)
-
-    where_clause = f'WHERE {" AND ".join(conditions)}' if conditions else ''
-    params.append(limit)
-
-    with get_db() as conn:
-        cursor = conn.execute(f'''
-            SELECT * FROM tscm_schedules
-            {where_clause}
-            ORDER BY id DESC
-            LIMIT ?
-        ''', params)
-        return [dict(row) for row in cursor]
-
-
-def update_tscm_schedule(schedule_id: int, **fields) -> bool:
-    """Update a TSCM schedule."""
-    if not fields:
-        return False
-
-    updates = []
-    params = []
-
-    for key, value in fields.items():
-        updates.append(f'{key} = ?')
-        params.append(value)
-
-    params.append(schedule_id)
-
-    with get_db() as conn:
-        cursor = conn.execute(
-            f'UPDATE tscm_schedules SET {", ".join(updates)} WHERE id = ?',
-            params
-        )
-        return cursor.rowcount > 0
-
-
-def delete_tscm_schedule(schedule_id: int) -> bool:
-    """Delete a TSCM schedule."""
-    with get_db() as conn:
-        cursor = conn.execute(
-            'DELETE FROM tscm_schedules WHERE id = ?',
-            (schedule_id,)
-        )
-        return cursor.rowcount > 0
-
-
-def is_known_good_device(identifier: str, location: str | None = None) -> dict | None:
-    """Check if a device is in the known-good registry for a location."""
-    with get_db() as conn:
-        if location:
-            cursor = conn.execute('''
+def delete_known_device(identifier: str) -> bool:
+    """Remove a device from the known-good registry."""
+    with get_db() as conn:
+        cursor = conn.execute(
+            'DELETE FROM tscm_known_devices WHERE identifier = ?',
+            (identifier.upper(),)
+        )
+        return cursor.rowcount > 0
+
+
+# =============================================================================
+# TSCM Schedule Functions
+# =============================================================================
+
+def create_tscm_schedule(
+    name: str,
+    cron_expression: str,
+    sweep_type: str = 'standard',
+    baseline_id: int | None = None,
+    zone_name: str | None = None,
+    enabled: bool = True,
+    notify_on_threat: bool = True,
+    notify_email: str | None = None,
+    last_run: str | None = None,
+    next_run: str | None = None,
+) -> int:
+    """Create a new TSCM sweep schedule."""
+    with get_db() as conn:
+        cursor = conn.execute('''
+            INSERT INTO tscm_schedules
+            (name, baseline_id, zone_name, cron_expression, sweep_type,
+             enabled, last_run, next_run, notify_on_threat, notify_email)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ''', (
+            name,
+            baseline_id,
+            zone_name,
+            cron_expression,
+            sweep_type,
+            1 if enabled else 0,
+            last_run,
+            next_run,
+            1 if notify_on_threat else 0,
+            notify_email,
+        ))
+        return cursor.lastrowid
+
+
+def get_tscm_schedule(schedule_id: int) -> dict | None:
+    """Get a TSCM schedule by ID."""
+    with get_db() as conn:
+        cursor = conn.execute(
+            'SELECT * FROM tscm_schedules WHERE id = ?',
+            (schedule_id,)
+        )
+        row = cursor.fetchone()
+        return dict(row) if row else None
+
+
+def get_all_tscm_schedules(
+    enabled: bool | None = None,
+    limit: int = 200
+) -> list[dict]:
+    """Get all TSCM schedules."""
+    conditions = []
+    params = []
+
+    if enabled is not None:
+        conditions.append('enabled = ?')
+        params.append(1 if enabled else 0)
+
+    where_clause = f'WHERE {" AND ".join(conditions)}' if conditions else ''
+    params.append(limit)
+
+    with get_db() as conn:
+        cursor = conn.execute(f'''
+            SELECT * FROM tscm_schedules
+            {where_clause}
+            ORDER BY id DESC
+            LIMIT ?
+        ''', params)
+        return [dict(row) for row in cursor]
+
+
+def update_tscm_schedule(schedule_id: int, **fields) -> bool:
+    """Update a TSCM schedule."""
+    if not fields:
+        return False
+
+    updates = []
+    params = []
+
+    for key, value in fields.items():
+        updates.append(f'{key} = ?')
+        params.append(value)
+
+    params.append(schedule_id)
+
+    with get_db() as conn:
+        cursor = conn.execute(
+            f'UPDATE tscm_schedules SET {", ".join(updates)} WHERE id = ?',
+            params
+        )
+        return cursor.rowcount > 0
+
+
+def delete_tscm_schedule(schedule_id: int) -> bool:
+    """Delete a TSCM schedule."""
+    with get_db() as conn:
+        cursor = conn.execute(
+            'DELETE FROM tscm_schedules WHERE id = ?',
+            (schedule_id,)
+        )
+        return cursor.rowcount > 0
+
+
+def is_known_good_device(identifier: str, location: str | None = None) -> dict | None:
+    """Check if a device is in the known-good registry for a location."""
+    with get_db() as conn:
+        if location:
+            cursor = conn.execute('''
                 SELECT * FROM tscm_known_devices
                 WHERE identifier = ? AND (location = ? OR scope = 'global')
             ''', (identifier.upper(), location))
@@ -2123,3 +2297,61 @@ def cleanup_old_payloads(max_age_hours: int = 24) -> int:
             WHERE received_at < datetime('now', ?)
         ''', (f'-{max_age_hours} hours',))
         return cursor.rowcount
+
+
+# =============================================================================
+# GSM Cleanup Functions
+# =============================================================================
+
+def cleanup_old_gsm_signals(max_age_days: int = 60) -> int:
+    """
+    Remove old GSM signal observations (60-day archive).
+
+    Args:
+        max_age_days: Maximum age in days (default: 60)
+
+    Returns:
+        Number of deleted entries
+    """
+    with get_db() as conn:
+        cursor = conn.execute('''
+            DELETE FROM gsm_signals
+            WHERE timestamp < datetime('now', ?)
+        ''', (f'-{max_age_days} days',))
+        return cursor.rowcount
+
+
+def cleanup_old_gsm_tmsi_log(max_age_hours: int = 24) -> int:
+    """
+    Remove old TMSI log entries (24-hour buffer for crowd density).
+
+    Args:
+        max_age_hours: Maximum age in hours (default: 24)
+
+    Returns:
+        Number of deleted entries
+    """
+    with get_db() as conn:
+        cursor = conn.execute('''
+            DELETE FROM gsm_tmsi_log
+            WHERE timestamp < datetime('now', ?)
+        ''', (f'-{max_age_hours} hours',))
+        return cursor.rowcount
+
+
+def cleanup_old_gsm_velocity_log(max_age_hours: int = 1) -> int:
+    """
+    Remove old velocity log entries (1-hour buffer for movement tracking).
+
+    Args:
+        max_age_hours: Maximum age in hours (default: 1)
+
+    Returns:
+        Number of deleted entries
+    """
+    with get_db() as conn:
+        cursor = conn.execute('''
+            DELETE FROM gsm_velocity_log
+            WHERE timestamp < datetime('now', ?)
+        ''', (f'-{max_age_hours} hours',))
+        return cursor.rowcount

utils/dependencies.py

@@ -443,6 +443,38 @@ TOOL_DEPENDENCIES = {
                 }
             }
         }
+    },
+    'gsm': {
+        'name': 'GSM Intelligence',
+        'tools': {
+            'grgsm_scanner': {
+                'required': True,
+                'description': 'gr-gsm scanner for finding GSM towers',
+                'install': {
+                    'apt': 'Build gr-gsm from source: https://github.com/ptrkrysik/gr-gsm',
+                    'brew': 'brew install gr-gsm (may require manual build)',
+                    'manual': 'https://github.com/ptrkrysik/gr-gsm'
+                }
+            },
+            'grgsm_livemon': {
+                'required': True,
+                'description': 'gr-gsm live monitor for decoding GSM signals',
+                'install': {
+                    'apt': 'Included with gr-gsm package',
+                    'brew': 'Included with gr-gsm',
+                    'manual': 'Included with gr-gsm'
+                }
+            },
+            'tshark': {
+                'required': True,
+                'description': 'Wireshark CLI for parsing GSM packets',
+                'install': {
+                    'apt': 'sudo apt-get install tshark',
+                    'brew': 'brew install wireshark',
+                    'manual': 'https://www.wireshark.org/download.html'
+                }
+            }
+        }
     }
 }
 

utils/gsm_geocoding.py

@@ -0,0 +1,200 @@
+"""GSM Cell Tower Geocoding Service.
+
+Provides hybrid cache-first geocoding with async API fallback for cell towers.
+"""
+
+from __future__ import annotations
+
+import logging
+import queue
+from typing import Any
+
+import requests
+
+import config
+from utils.database import get_db
+
+logger = logging.getLogger('intercept.gsm_geocoding')
+
+# Queue for pending geocoding requests
+_geocoding_queue = queue.Queue(maxsize=100)
+
+
+def lookup_cell_coordinates(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None:
+    """
+    Lookup cell tower coordinates with cache-first strategy.
+
+    Strategy:
+    1. Check gsm_cells table (cache) - fast synchronous lookup
+    2. If not found, return None (caller decides whether to use API)
+
+    Args:
+        mcc: Mobile Country Code
+        mnc: Mobile Network Code
+        lac: Location Area Code
+        cid: Cell ID
+
+    Returns:
+        dict with keys: lat, lon, source='cache', azimuth (optional),
+        range_meters (optional), operator (optional), radio (optional)
+        Returns None if not found in cache.
+    """
+    try:
+        with get_db() as conn:
+            result = conn.execute('''
+                SELECT lat, lon, azimuth, range_meters, operator, radio
+                FROM gsm_cells
+                WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ?
+            ''', (mcc, mnc, lac, cid)).fetchone()
+
+            if result:
+                return {
+                    'lat': result['lat'],
+                    'lon': result['lon'],
+                    'source': 'cache',
+                    'azimuth': result['azimuth'],
+                    'range_meters': result['range_meters'],
+                    'operator': result['operator'],
+                    'radio': result['radio']
+                }
+
+        return None
+
+    except Exception as e:
+        logger.error(f"Error looking up coordinates from cache: {e}")
+        return None
+
+
+def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None:
+    """
+    Lookup cell tower from OpenCellID API and cache result.
+
+    Args:
+        mcc: Mobile Country Code
+        mnc: Mobile Network Code
+        lac: Location Area Code
+        cid: Cell ID
+
+    Returns:
+        dict with keys: lat, lon, source='api', azimuth (optional),
+        range_meters (optional), operator (optional), radio (optional)
+        Returns None if API call fails or cell not found.
+    """
+    try:
+        api_url = config.GSM_OPENCELLID_API_URL
+        params = {
+            'key': config.GSM_OPENCELLID_API_KEY,
+            'mcc': mcc,
+            'mnc': mnc,
+            'lac': lac,
+            'cellid': cid,
+            'format': 'json'
+        }
+
+        response = requests.get(api_url, params=params, timeout=10)
+
+        if response.status_code == 200:
+            cell_data = response.json()
+
+            # Cache the result
+            with get_db() as conn:
+                conn.execute('''
+                    INSERT OR REPLACE INTO gsm_cells
+                    (mcc, mnc, lac, cid, lat, lon, azimuth, range_meters, samples, radio, operator, last_verified)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
+                ''', (
+                    mcc, mnc, lac, cid,
+                    cell_data.get('lat'),
+                    cell_data.get('lon'),
+                    cell_data.get('azimuth'),
+                    cell_data.get('range'),
+                    cell_data.get('samples'),
+                    cell_data.get('radio'),
+                    cell_data.get('operator')
+                ))
+                conn.commit()
+
+            logger.info(f"Cached cell tower from API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
+
+            return {
+                'lat': cell_data.get('lat'),
+                'lon': cell_data.get('lon'),
+                'source': 'api',
+                'azimuth': cell_data.get('azimuth'),
+                'range_meters': cell_data.get('range'),
+                'operator': cell_data.get('operator'),
+                'radio': cell_data.get('radio')
+            }
+        else:
+            logger.warning(f"OpenCellID API returned {response.status_code} for MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
+            return None
+
+    except Exception as e:
+        logger.error(f"Error calling OpenCellID API: {e}")
+        return None
+
+
+def enrich_tower_data(tower_data: dict[str, Any]) -> dict[str, Any]:
+    """
+    Enrich tower data with coordinates using cache-first strategy.
+
+    If coordinates found in cache, adds them immediately.
+    If not found, marks as 'pending' and queues for background API lookup.
+
+    Args:
+        tower_data: Dictionary with keys mcc, mnc, lac, cid (and other tower data)
+
+    Returns:
+        Enriched tower_data dict with added fields:
+        - lat, lon (if found in cache)
+        - status='pending' (if needs API lookup)
+        - source='cache' (if from cache)
+    """
+    mcc = tower_data.get('mcc')
+    mnc = tower_data.get('mnc')
+    lac = tower_data.get('lac')
+    cid = tower_data.get('cid')
+
+    # Validate required fields
+    if not all([mcc is not None, mnc is not None, lac is not None, cid is not None]):
+        logger.warning(f"Tower data missing required fields: {tower_data}")
+        return tower_data
+
+    # Try cache lookup
+    coords = lookup_cell_coordinates(mcc, mnc, lac, cid)
+
+    if coords:
+        # Found in cache - add coordinates immediately
+        tower_data['lat'] = coords['lat']
+        tower_data['lon'] = coords['lon']
+        tower_data['source'] = 'cache'
+
+        # Add optional fields if available
+        if coords.get('azimuth') is not None:
+            tower_data['azimuth'] = coords['azimuth']
+        if coords.get('range_meters') is not None:
+            tower_data['range_meters'] = coords['range_meters']
+        if coords.get('operator'):
+            tower_data['operator'] = coords['operator']
+        if coords.get('radio'):
+            tower_data['radio'] = coords['radio']
+
+        logger.debug(f"Cache hit for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
+    else:
+        # Not in cache - mark as pending and queue for API lookup
+        tower_data['status'] = 'pending'
+        tower_data['source'] = 'unknown'
+
+        # Queue for background geocoding (non-blocking)
+        try:
+            _geocoding_queue.put_nowait(tower_data.copy())
+            logger.debug(f"Queued tower for geocoding: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}")
+        except queue.Full:
+            logger.warning("Geocoding queue full, dropping tower")
+
+    return tower_data
+
+
+def get_geocoding_queue() -> queue.Queue:
+    """Get the geocoding queue for the background worker."""
+    return _geocoding_queue