diff --git a/app.py b/app.py index d89f9c6..76ee903 100644 --- a/app.py +++ b/app.py @@ -39,6 +39,7 @@ from utils.constants import ( MAX_VESSEL_AGE_SECONDS, MAX_DSC_MESSAGE_AGE_SECONDS, MAX_DEAUTH_ALERTS_AGE_SECONDS, + MAX_GSM_AGE_SECONDS, QUEUE_MAX_SIZE, ) import logging @@ -187,6 +188,16 @@ deauth_detector = None deauth_detector_queue = queue.Queue(maxsize=QUEUE_MAX_SIZE) deauth_detector_lock = threading.Lock() +# GSM Spy +gsm_spy_scanner_running = False # Flag: scanner thread active +gsm_spy_livemon_process = None # For grgsm_livemon process +gsm_spy_monitor_process = None # For tshark monitoring process +gsm_spy_queue = queue.Queue(maxsize=QUEUE_MAX_SIZE) +gsm_spy_lock = threading.Lock() +gsm_spy_active_device = None +gsm_spy_selected_arfcn = None +gsm_spy_region = 'Americas' # Default band + # ============================================ # GLOBAL STATE DICTIONARIES # ============================================ @@ -219,6 +230,16 @@ dsc_messages = DataStore(max_age_seconds=MAX_DSC_MESSAGE_AGE_SECONDS, name='dsc_ # Deauth alerts - using DataStore for automatic cleanup deauth_alerts = DataStore(max_age_seconds=MAX_DEAUTH_ALERTS_AGE_SECONDS, name='deauth_alerts') +# GSM Spy data stores +gsm_spy_towers = DataStore( + max_age_seconds=MAX_GSM_AGE_SECONDS, + name='gsm_spy_towers' +) +gsm_spy_devices = DataStore( + max_age_seconds=MAX_GSM_AGE_SECONDS, + name='gsm_spy_devices' +) + # Satellite state satellite_passes = [] # Predicted satellite passes (not auto-cleaned, calculated) @@ -231,6 +252,8 @@ cleanup_manager.register(adsb_aircraft) cleanup_manager.register(ais_vessels) cleanup_manager.register(dsc_messages) cleanup_manager.register(deauth_alerts) +cleanup_manager.register(gsm_spy_towers) +cleanup_manager.register(gsm_spy_devices) # ============================================ # SDR DEVICE REGISTRY @@ -664,6 +687,8 @@ def kill_all() -> Response: global current_process, sensor_process, wifi_process, adsb_process, ais_process, acars_process global aprs_process, aprs_rtl_process, dsc_process, dsc_rtl_process, bt_process global dmr_process, dmr_rtl_process + global gsm_spy_livemon_process, gsm_spy_monitor_process + global gsm_spy_scanner_running, gsm_spy_active_device, gsm_spy_selected_arfcn, gsm_spy_region # Import adsb and ais modules to reset their state from routes import adsb as adsb_module @@ -677,6 +702,7 @@ def kill_all() -> Response: 'dump1090', 'acarsdec', 'direwolf', 'AIS-catcher', 'hcitool', 'bluetoothctl', 'dsd', 'rtl_tcp', 'rtl_power', 'rtlamr', 'ffmpeg', + 'grgsm_scanner', 'grgsm_livemon', 'tshark' ] for proc in processes_to_kill: @@ -745,6 +771,29 @@ def kill_all() -> Response: except Exception: pass + # Reset GSM Spy state + with gsm_spy_lock: + gsm_spy_scanner_running = False + gsm_spy_active_device = None + gsm_spy_selected_arfcn = None + gsm_spy_region = 'Americas' + + if gsm_spy_livemon_process: + try: + if safe_terminate(gsm_spy_livemon_process): + killed.append('grgsm_livemon') + except Exception: + pass + gsm_spy_livemon_process = None + + if gsm_spy_monitor_process: + try: + if safe_terminate(gsm_spy_monitor_process): + killed.append('tshark') + except Exception: + pass + gsm_spy_monitor_process = None + # Clear SDR device registry with sdr_device_registry_lock: sdr_device_registry.clear() @@ -834,6 +883,26 @@ def main() -> None: from utils.database import init_db init_db() + # Register database cleanup functions + from utils.database import ( + cleanup_old_gsm_signals, + cleanup_old_gsm_tmsi_log, + cleanup_old_gsm_velocity_log, + cleanup_old_signal_history, + cleanup_old_timeline_entries, + cleanup_old_dsc_alerts, + cleanup_old_payloads + ) + # GSM cleanups: signals (60 days), TMSI log (24 hours), velocity (1 hour) + # Interval multiplier: cleanup every N cycles (60s interval = 1 cleanup per hour at multiplier 60) + cleanup_manager.register_db_cleanup(cleanup_old_gsm_tmsi_log, interval_multiplier=60) # Every hour + cleanup_manager.register_db_cleanup(cleanup_old_gsm_velocity_log, interval_multiplier=60) # Every hour + cleanup_manager.register_db_cleanup(cleanup_old_gsm_signals, interval_multiplier=1440) # Every 24 hours + cleanup_manager.register_db_cleanup(cleanup_old_signal_history, interval_multiplier=1440) # Every 24 hours + cleanup_manager.register_db_cleanup(cleanup_old_timeline_entries, interval_multiplier=1440) # Every 24 hours + cleanup_manager.register_db_cleanup(cleanup_old_dsc_alerts, interval_multiplier=1440) # Every 24 hours + cleanup_manager.register_db_cleanup(cleanup_old_payloads, interval_multiplier=1440) # Every 24 hours + # Start automatic cleanup of stale data entries cleanup_manager.start() diff --git a/config.py b/config.py index 269c81b..98f2d1d 100644 --- a/config.py +++ b/config.py @@ -218,6 +218,12 @@ ALERT_WEBHOOK_TIMEOUT = _get_env_int('ALERT_WEBHOOK_TIMEOUT', 5) ADMIN_USERNAME = _get_env('ADMIN_USERNAME', 'admin') ADMIN_PASSWORD = _get_env('ADMIN_PASSWORD', 'admin') +# GSM Spy settings +GSM_OPENCELLID_API_KEY = _get_env('GSM_OPENCELLID_API_KEY', '') +GSM_OPENCELLID_API_URL = _get_env('GSM_OPENCELLID_API_URL', 'https://opencellid.org/cell/get') +GSM_API_DAILY_LIMIT = _get_env_int('GSM_API_DAILY_LIMIT', 1000) +GSM_TA_METERS_PER_UNIT = _get_env_int('GSM_TA_METERS_PER_UNIT', 554) + def configure_logging() -> None: """Configure application logging.""" logging.basicConfig( diff --git a/routes/__init__.py b/routes/__init__.py index 448d771..af211cc 100644 --- a/routes/__init__.py +++ b/routes/__init__.py @@ -27,10 +27,11 @@ def register_blueprints(app): from .updater import updater_bp from .sstv import sstv_bp from .sstv_general import sstv_general_bp - from .dmr import dmr_bp - from .websdr import websdr_bp - from .alerts import alerts_bp - from .recordings import recordings_bp + from .dmr import dmr_bp + from .websdr import websdr_bp + from .alerts import alerts_bp + from .recordings import recordings_bp + from .gsm_spy import gsm_spy_bp app.register_blueprint(pager_bp) app.register_blueprint(sensor_bp) @@ -57,10 +58,11 @@ def register_blueprints(app): app.register_blueprint(updater_bp) # GitHub update checking app.register_blueprint(sstv_bp) # ISS SSTV decoder app.register_blueprint(sstv_general_bp) # General terrestrial SSTV - app.register_blueprint(dmr_bp) # DMR / P25 / Digital Voice - app.register_blueprint(websdr_bp) # HF/Shortwave WebSDR - app.register_blueprint(alerts_bp) # Cross-mode alerts - app.register_blueprint(recordings_bp) # Session recordings + app.register_blueprint(dmr_bp) # DMR / P25 / Digital Voice + app.register_blueprint(websdr_bp) # HF/Shortwave WebSDR + app.register_blueprint(alerts_bp) # Cross-mode alerts + app.register_blueprint(recordings_bp) # Session recordings + app.register_blueprint(gsm_spy_bp) # GSM cellular intelligence # Initialize TSCM state with queue and lock from app import app as app_module diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py new file mode 100644 index 0000000..2166e77 --- /dev/null +++ b/routes/gsm_spy.py @@ -0,0 +1,1511 @@ +"""GSM Spy route handlers for cellular tower and device tracking.""" + +from __future__ import annotations + +import json +import logging +import queue +import re +import subprocess +import threading +import time +from datetime import datetime, timedelta +from typing import Any + +import requests +from flask import Blueprint, Response, jsonify, render_template, request + +import app as app_module +import config +from config import SHARED_OBSERVER_LOCATION_ENABLED +from utils.database import get_db +from utils.process import register_process, safe_terminate, unregister_process +from utils.sse import format_sse +from utils.validation import validate_device_index + +logger = logging.getLogger('intercept.gsm_spy') + +gsm_spy_bp = Blueprint('gsm_spy', __name__, url_prefix='/gsm_spy') + +# Regional band configurations (G-01) +REGIONAL_BANDS = { + 'Americas': { + 'GSM850': {'start': 869e6, 'end': 894e6, 'arfcn_start': 128, 'arfcn_end': 251}, + 'PCS1900': {'start': 1930e6, 'end': 1990e6, 'arfcn_start': 512, 'arfcn_end': 810} + }, + 'Europe': { + 'GSM800': {'start': 832e6, 'end': 862e6, 'arfcn_start': 438, 'arfcn_end': 511}, # E-GSM800 downlink + 'GSM850': {'start': 869e6, 'end': 894e6, 'arfcn_start': 128, 'arfcn_end': 251}, # Also used in some EU countries + 'EGSM900': {'start': 925e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, + 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} + }, + 'Asia': { + 'EGSM900': {'start': 925e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, + 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} + } +} + +# Module state tracking +gsm_using_service = False +gsm_connected = False +gsm_towers_found = 0 +gsm_devices_tracked = 0 + +# Geocoding worker state +_geocoding_worker_thread = None + + +# ============================================ +# API Usage Tracking Helper Functions +# ============================================ + +def get_api_usage_today(): + """Get OpenCellID API usage count for today.""" + from utils.database import get_setting + today = datetime.now().date().isoformat() + usage_date = get_setting('gsm.opencellid.usage_date', '') + + # Reset counter if new day + if usage_date != today: + from utils.database import set_setting + set_setting('gsm.opencellid.usage_date', today) + set_setting('gsm.opencellid.usage_count', 0) + return 0 + + return get_setting('gsm.opencellid.usage_count', 0) + + +def increment_api_usage(): + """Increment OpenCellID API usage counter.""" + from utils.database import set_setting + current = get_api_usage_today() + set_setting('gsm.opencellid.usage_count', current + 1) + return current + 1 + + +def can_use_api(): + """Check if we can make an API call within daily limit.""" + current_usage = get_api_usage_today() + return current_usage < config.GSM_API_DAILY_LIMIT + + +# ============================================ +# Background Geocoding Worker +# ============================================ + +def start_geocoding_worker(): + """Start background thread for async geocoding.""" + global _geocoding_worker_thread + if _geocoding_worker_thread is None or not _geocoding_worker_thread.is_alive(): + _geocoding_worker_thread = threading.Thread( + target=geocoding_worker, + daemon=True, + name='gsm-geocoding-worker' + ) + _geocoding_worker_thread.start() + logger.info("Started geocoding worker thread") + + +def geocoding_worker(): + """Worker thread processes pending geocoding requests.""" + from utils.gsm_geocoding import lookup_cell_from_api, get_geocoding_queue + + geocoding_queue = get_geocoding_queue() + + while True: + try: + # Wait for pending tower with timeout + tower_data = geocoding_queue.get(timeout=5) + + # Check rate limit + if not can_use_api(): + current_usage = get_api_usage_today() + logger.warning(f"OpenCellID API rate limit reached ({current_usage}/{config.GSM_API_DAILY_LIMIT})") + geocoding_queue.task_done() + continue + + # Call API + mcc = tower_data.get('mcc') + mnc = tower_data.get('mnc') + lac = tower_data.get('lac') + cid = tower_data.get('cid') + + logger.debug(f"Geocoding tower via API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") + + coords = lookup_cell_from_api(mcc, mnc, lac, cid) + + if coords: + # Update tower data with coordinates + tower_data['lat'] = coords['lat'] + tower_data['lon'] = coords['lon'] + tower_data['source'] = 'api' + tower_data['status'] = 'resolved' + tower_data['type'] = 'tower_update' + + # Add optional fields if available + if coords.get('azimuth') is not None: + tower_data['azimuth'] = coords['azimuth'] + if coords.get('range_meters') is not None: + tower_data['range_meters'] = coords['range_meters'] + if coords.get('operator'): + tower_data['operator'] = coords['operator'] + if coords.get('radio'): + tower_data['radio'] = coords['radio'] + + # Update DataStore + key = f"{mcc}_{mnc}_{lac}_{cid}" + app_module.gsm_spy_towers[key] = tower_data + + # Send update to SSE stream + try: + app_module.gsm_spy_queue.put_nowait(tower_data) + logger.info(f"Resolved coordinates for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") + except queue.Full: + logger.warning("SSE queue full, dropping tower update") + + # Increment API usage counter + usage_count = increment_api_usage() + logger.info(f"OpenCellID API call #{usage_count} today") + + else: + logger.warning(f"Could not resolve coordinates for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") + + geocoding_queue.task_done() + + # Rate limiting between API calls (be nice to OpenCellID) + time.sleep(1) + + except queue.Empty: + # No pending towers, continue waiting + continue + except Exception as e: + logger.error(f"Geocoding worker error: {e}", exc_info=True) + time.sleep(1) + + +def arfcn_to_frequency(arfcn): + """Convert ARFCN to downlink frequency in Hz. + + Uses REGIONAL_BANDS to determine the correct band and conversion formula. + Returns frequency in Hz (e.g., 925800000 for 925.8 MHz). + """ + arfcn = int(arfcn) + + # Search all bands to find which one this ARFCN belongs to + for region_bands in REGIONAL_BANDS.values(): + for band_name, band_info in region_bands.items(): + arfcn_start = band_info['arfcn_start'] + arfcn_end = band_info['arfcn_end'] + + if arfcn_start <= arfcn <= arfcn_end: + # Found the right band, calculate frequency + # Downlink frequency = band_start + (arfcn - arfcn_start) * 200kHz + freq_hz = band_info['start'] + (arfcn - arfcn_start) * 200000 + return int(freq_hz) + + # If ARFCN not found in any band, raise error + raise ValueError(f"ARFCN {arfcn} not found in any known GSM band") + + +def validate_band_names(bands: list[str], region: str) -> tuple[list[str], str | None]: + """Validate band names against REGIONAL_BANDS whitelist. + + Args: + bands: List of band names from user input + region: Region name (Americas, Europe, Asia) + + Returns: + Tuple of (validated_bands, error_message) + """ + if not bands: + return [], None + + region_bands = REGIONAL_BANDS.get(region) + if not region_bands: + return [], f"Invalid region: {region}" + + valid_band_names = set(region_bands.keys()) + invalid_bands = [b for b in bands if b not in valid_band_names] + + if invalid_bands: + return [], (f"Invalid bands for {region}: {', '.join(invalid_bands)}. " + f"Valid bands: {', '.join(sorted(valid_band_names))}") + + return bands, None + + +def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subprocess.Popen, subprocess.Popen]: + """Start grgsm_livemon and tshark processes for monitoring an ARFCN. + + Returns: + Tuple of (grgsm_process, tshark_process) + """ + frequency_hz = arfcn_to_frequency(arfcn) + frequency_mhz = frequency_hz / 1e6 + + # Start grgsm_livemon + grgsm_cmd = [ + 'grgsm_livemon', + '--args', f'rtl={device_index}', + '-f', f'{frequency_mhz}M' + ] + grgsm_proc = subprocess.Popen( + grgsm_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + register_process(grgsm_proc) + logger.info(f"Started grgsm_livemon (PID: {grgsm_proc.pid})") + + time.sleep(2) # Wait for grgsm_livemon to start + + # Start tshark + tshark_cmd = [ + 'tshark', '-i', 'lo', + '-Y', 'gsm_a.rr.timing_advance || gsm_a.tmsi || gsm_a.imsi', + '-T', 'fields', + '-e', 'gsm_a.rr.timing_advance', + '-e', 'gsm_a.tmsi', + '-e', 'gsm_a.imsi', + '-e', 'gsm_a.lac', + '-e', 'gsm_a.cellid' + ] + tshark_proc = subprocess.Popen( + tshark_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + bufsize=1 + ) + register_process(tshark_proc) + logger.info(f"Started tshark (PID: {tshark_proc.pid})") + + return grgsm_proc, tshark_proc + + +def _start_and_register_monitor(arfcn: int, device_index: int) -> None: + """Start monitoring processes and register them in global state. + + This is shared logic between start_monitor() and auto_start_monitor(). + Must be called within gsm_spy_lock context. + + Args: + arfcn: ARFCN to monitor + device_index: SDR device index + """ + # Start monitoring processes + grgsm_proc, tshark_proc = _start_monitoring_processes(arfcn, device_index) + app_module.gsm_spy_livemon_process = grgsm_proc + app_module.gsm_spy_monitor_process = tshark_proc + app_module.gsm_spy_selected_arfcn = arfcn + + # Start monitoring thread + monitor_thread_obj = threading.Thread( + target=monitor_thread, + args=(tshark_proc,), + daemon=True + ) + monitor_thread_obj.start() + + +@gsm_spy_bp.route('/dashboard') +def dashboard(): + """Render GSM Spy dashboard.""" + return render_template( + 'gsm_spy_dashboard.html', + shared_observer_location=SHARED_OBSERVER_LOCATION_ENABLED + ) + + +@gsm_spy_bp.route('/start', methods=['POST']) +def start_scanner(): + """Start GSM scanner (G-01 BTS Scanner).""" + global gsm_towers_found, gsm_connected + + with app_module.gsm_spy_lock: + if app_module.gsm_spy_scanner_running: + return jsonify({'error': 'Scanner already running'}), 400 + + data = request.get_json() or {} + device_index = data.get('device', 0) + region = data.get('region', 'Americas') + selected_bands = data.get('bands', []) # Get user-selected bands + + # Validate device index + try: + device_index = validate_device_index(device_index) + except ValueError as e: + return jsonify({'error': str(e)}), 400 + + # Claim SDR device to prevent conflicts + from app import claim_sdr_device + claim_error = claim_sdr_device(device_index, 'GSM Spy') + if claim_error: + return jsonify({ + 'error': claim_error, + 'error_type': 'DEVICE_BUSY' + }), 409 + + # If no bands selected, use all bands for the region (backwards compatibility) + if selected_bands: + validated_bands, error = validate_band_names(selected_bands, region) + if error: + from app import release_sdr_device + release_sdr_device(device_index) + return jsonify({'error': error}), 400 + selected_bands = validated_bands + else: + region_bands = REGIONAL_BANDS.get(region, REGIONAL_BANDS['Americas']) + selected_bands = list(region_bands.keys()) + logger.warning(f"No bands specified, using all bands for {region}: {selected_bands}") + + # Build grgsm_scanner command + # Example: grgsm_scanner --args="rtl=0" -b GSM900 + try: + cmd = ['grgsm_scanner'] + + # Add device argument (--args for RTL-SDR device selection) + cmd.extend(['--args', f'rtl={device_index}']) + + # Add selected band arguments + # Map EGSM900 to GSM900 since that's what grgsm_scanner expects + for band_name in selected_bands: + # Normalize band name (EGSM900 -> GSM900, remove EGSM prefix) + normalized_band = band_name.replace('EGSM', 'GSM') + cmd.extend(['-b', normalized_band]) + + logger.info(f"Starting GSM scanner: {' '.join(cmd)}") + + # Set a flag to indicate scanner should run + app_module.gsm_spy_active_device = device_index + app_module.gsm_spy_region = region + app_module.gsm_spy_scanner_running = True # Use as flag initially + + # Reset counters for new session + gsm_towers_found = 0 + gsm_devices_tracked = 0 + + # Start geocoding worker (if not already running) + start_geocoding_worker() + + # Start scanning thread (will run grgsm_scanner in a loop) + scanner_thread_obj = threading.Thread( + target=scanner_thread, + args=(cmd, device_index), + daemon=True + ) + scanner_thread_obj.start() + + gsm_connected = True + + return jsonify({ + 'status': 'started', + 'device': device_index, + 'region': region + }) + + except FileNotFoundError: + from app import release_sdr_device + release_sdr_device(device_index) + return jsonify({'error': 'grgsm_scanner not found. Please install gr-gsm.'}), 500 + except Exception as e: + from app import release_sdr_device + release_sdr_device(device_index) + logger.error(f"Error starting GSM scanner: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/monitor', methods=['POST']) +def start_monitor(): + """Start monitoring specific tower (G-02 Decoding).""" + with app_module.gsm_spy_lock: + if app_module.gsm_spy_monitor_process: + return jsonify({'error': 'Monitor already running'}), 400 + + data = request.get_json() or {} + arfcn = data.get('arfcn') + device_index = data.get('device', app_module.gsm_spy_active_device or 0) + + if not arfcn: + return jsonify({'error': 'ARFCN required'}), 400 + + # Validate ARFCN is valid integer and in known GSM band ranges + try: + arfcn = int(arfcn) + # This will raise ValueError if ARFCN is not in any known band + arfcn_to_frequency(arfcn) + except (ValueError, TypeError) as e: + return jsonify({'error': f'Invalid ARFCN: {e}'}), 400 + + # Validate device index + try: + device_index = validate_device_index(device_index) + except ValueError as e: + return jsonify({'error': str(e)}), 400 + + try: + # Start and register monitoring (shared logic) + _start_and_register_monitor(arfcn, device_index) + + return jsonify({ + 'status': 'monitoring', + 'arfcn': arfcn, + 'device': device_index + }) + + except FileNotFoundError as e: + return jsonify({'error': f'Tool not found: {e}'}), 500 + except Exception as e: + logger.error(f"Error starting monitor: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/stop', methods=['POST']) +def stop_scanner(): + """Stop GSM scanner and monitor.""" + global gsm_connected, gsm_towers_found, gsm_devices_tracked + + with app_module.gsm_spy_lock: + killed = [] + + # Stop scanner (now just a flag, thread will see it and exit) + if app_module.gsm_spy_scanner_running: + app_module.gsm_spy_scanner_running = False + killed.append('scanner') + + # Terminate livemon process + if app_module.gsm_spy_livemon_process: + unregister_process(app_module.gsm_spy_livemon_process) + if safe_terminate(app_module.gsm_spy_livemon_process, timeout=5): + killed.append('livemon') + app_module.gsm_spy_livemon_process = None + + # Terminate monitor process + if app_module.gsm_spy_monitor_process: + unregister_process(app_module.gsm_spy_monitor_process) + if safe_terminate(app_module.gsm_spy_monitor_process, timeout=5): + killed.append('monitor') + app_module.gsm_spy_monitor_process = None + + # Note: SDR device is released by scanner thread's finally block + # to avoid race condition. Just reset the state variables here. + app_module.gsm_spy_active_device = None + app_module.gsm_spy_selected_arfcn = None + gsm_connected = False + gsm_towers_found = 0 + gsm_devices_tracked = 0 + + return jsonify({'status': 'stopped', 'killed': killed}) + + +@gsm_spy_bp.route('/stream') +def stream(): + """SSE stream for real-time GSM updates.""" + def generate(): + """Generate SSE events.""" + last_keepalive = time.time() + + while True: + try: + # Check if scanner is still running + if not app_module.gsm_spy_scanner_running and not app_module.gsm_spy_monitor_process: + yield format_sse({'type': 'disconnected'}) + break + + # Try to get data from queue + try: + data = app_module.gsm_spy_queue.get(timeout=1) + yield format_sse(data) + last_keepalive = time.time() + except queue.Empty: + # Send keepalive if needed + if time.time() - last_keepalive > 30: + yield format_sse({'type': 'keepalive'}) + last_keepalive = time.time() + + except GeneratorExit: + break + except Exception as e: + logger.error(f"Error in GSM stream: {e}") + yield format_sse({'type': 'error', 'message': str(e)}) + break + + return Response( + generate(), + mimetype='text/event-stream', + headers={ + 'Cache-Control': 'no-cache', + 'X-Accel-Buffering': 'no' + } + ) + + +@gsm_spy_bp.route('/status') +def status(): + """Get current GSM Spy status.""" + api_usage = get_api_usage_today() + return jsonify({ + 'running': bool(app_module.gsm_spy_scanner_running), + 'monitoring': app_module.gsm_spy_monitor_process is not None, + 'towers_found': gsm_towers_found, + 'devices_tracked': gsm_devices_tracked, + 'device': app_module.gsm_spy_active_device, + 'region': app_module.gsm_spy_region, + 'selected_arfcn': app_module.gsm_spy_selected_arfcn, + 'api_usage_today': api_usage, + 'api_limit': config.GSM_API_DAILY_LIMIT, + 'api_remaining': config.GSM_API_DAILY_LIMIT - api_usage + }) + + +@gsm_spy_bp.route('/lookup_cell', methods=['POST']) +def lookup_cell(): + """Lookup cell tower via OpenCellID (G-05).""" + data = request.get_json() or {} + mcc = data.get('mcc') + mnc = data.get('mnc') + lac = data.get('lac') + cid = data.get('cid') + + if not all([mcc, mnc, lac, cid]): + return jsonify({'error': 'MCC, MNC, LAC, and CID required'}), 400 + + try: + # Check local cache first + with get_db() as conn: + result = conn.execute(''' + SELECT lat, lon, azimuth, range_meters, operator, radio + FROM gsm_cells + WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? + ''', (mcc, mnc, lac, cid)).fetchone() + + if result: + return jsonify({ + 'source': 'cache', + 'lat': result['lat'], + 'lon': result['lon'], + 'azimuth': result['azimuth'], + 'range': result['range_meters'], + 'operator': result['operator'], + 'radio': result['radio'] + }) + + # Check API usage limit + if not can_use_api(): + current_usage = get_api_usage_today() + return jsonify({ + 'error': 'OpenCellID API daily limit reached', + 'usage_today': current_usage, + 'limit': config.GSM_API_DAILY_LIMIT + }), 429 + + # Call OpenCellID API + api_url = config.GSM_OPENCELLID_API_URL + params = { + 'key': config.GSM_OPENCELLID_API_KEY, + 'mcc': mcc, + 'mnc': mnc, + 'lac': lac, + 'cellid': cid, + 'format': 'json' + } + + response = requests.get(api_url, params=params, timeout=10) + + if response.status_code == 200: + cell_data = response.json() + + # Increment API usage counter + usage_count = increment_api_usage() + logger.info(f"OpenCellID API call #{usage_count} today") + + # Cache the result + conn.execute(''' + INSERT OR REPLACE INTO gsm_cells + (mcc, mnc, lac, cid, lat, lon, azimuth, range_meters, samples, radio, operator, last_verified) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + ''', ( + mcc, mnc, lac, cid, + cell_data.get('lat'), + cell_data.get('lon'), + cell_data.get('azimuth'), + cell_data.get('range'), + cell_data.get('samples'), + cell_data.get('radio'), + cell_data.get('operator') + )) + conn.commit() + + return jsonify({ + 'source': 'api', + 'lat': cell_data.get('lat'), + 'lon': cell_data.get('lon'), + 'azimuth': cell_data.get('azimuth'), + 'range': cell_data.get('range'), + 'operator': cell_data.get('operator'), + 'radio': cell_data.get('radio') + }) + else: + return jsonify({'error': 'Cell not found in OpenCellID'}), 404 + + except Exception as e: + logger.error(f"Error looking up cell: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/detect_rogue', methods=['POST']) +def detect_rogue(): + """Analyze and flag rogue towers (G-07).""" + data = request.get_json() or {} + tower_info = data.get('tower') + + if not tower_info: + return jsonify({'error': 'Tower info required'}), 400 + + try: + is_rogue = False + reasons = [] + + # Check if tower exists in OpenCellID + mcc = tower_info.get('mcc') + mnc = tower_info.get('mnc') + lac = tower_info.get('lac') + cid = tower_info.get('cid') + + if all([mcc, mnc, lac, cid]): + with get_db() as conn: + result = conn.execute(''' + SELECT id FROM gsm_cells + WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? + ''', (mcc, mnc, lac, cid)).fetchone() + + if not result: + is_rogue = True + reasons.append('Tower not found in OpenCellID database') + + # Check signal strength anomalies + signal = tower_info.get('signal_strength', 0) + if signal > -50: # Suspiciously strong signal + is_rogue = True + reasons.append(f'Unusually strong signal: {signal} dBm') + + # If rogue, insert into database + if is_rogue: + with get_db() as conn: + conn.execute(''' + INSERT INTO gsm_rogues + (arfcn, mcc, mnc, lac, cid, signal_strength, reason, threat_level) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ''', ( + tower_info.get('arfcn'), + mcc, mnc, lac, cid, + signal, + '; '.join(reasons), + 'high' if len(reasons) > 1 else 'medium' + )) + conn.commit() + + return jsonify({ + 'is_rogue': is_rogue, + 'reasons': reasons + }) + + except Exception as e: + logger.error(f"Error detecting rogue: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/towers') +def get_towers(): + """Get all detected towers.""" + towers = [] + for key, tower_data in app_module.gsm_spy_towers.items(): + towers.append(tower_data) + return jsonify(towers) + + +@gsm_spy_bp.route('/devices') +def get_devices(): + """Get all tracked devices (IMSI/TMSI).""" + devices = [] + for key, device_data in app_module.gsm_spy_devices.items(): + devices.append(device_data) + return jsonify(devices) + + +@gsm_spy_bp.route('/rogues') +def get_rogues(): + """Get all detected rogue towers.""" + try: + with get_db() as conn: + results = conn.execute(''' + SELECT * FROM gsm_rogues + WHERE acknowledged = 0 + ORDER BY detected_at DESC + LIMIT 50 + ''').fetchall() + + rogues = [dict(row) for row in results] + return jsonify(rogues) + except Exception as e: + logger.error(f"Error fetching rogues: {e}") + return jsonify({'error': str(e)}), 500 + + +# ============================================ +# Advanced Features (G-08 through G-12) +# ============================================ + +@gsm_spy_bp.route('/velocity', methods=['GET']) +def get_velocity_data(): + """Get velocity vectoring data for tracked devices (G-08).""" + try: + device_id = request.args.get('device_id') + minutes = int(request.args.get('minutes', 60)) # Last 60 minutes by default + + with get_db() as conn: + # Get velocity log entries + query = ''' + SELECT * FROM gsm_velocity_log + WHERE timestamp >= datetime('now', '-' || ? || ' minutes') + ''' + params = [minutes] + + if device_id: + query += ' AND device_id = ?' + params.append(device_id) + + query += ' ORDER BY timestamp DESC LIMIT 100' + + results = conn.execute(query, params).fetchall() + velocity_data = [dict(row) for row in results] + + return jsonify(velocity_data) + except Exception as e: + logger.error(f"Error fetching velocity data: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/velocity/calculate', methods=['POST']) +def calculate_velocity(): + """Calculate velocity for a device based on TA transitions (G-08).""" + data = request.get_json() or {} + device_id = data.get('device_id') + + if not device_id: + return jsonify({'error': 'device_id required'}), 400 + + try: + with get_db() as conn: + # Get last two TA readings for this device + results = conn.execute(''' + SELECT ta_value, cid, timestamp + FROM gsm_signals + WHERE (imsi = ? OR tmsi = ?) + ORDER BY timestamp DESC + LIMIT 2 + ''', (device_id, device_id)).fetchall() + + if len(results) < 2: + return jsonify({'velocity': 0, 'message': 'Insufficient data'}) + + curr = dict(results[0]) + prev = dict(results[1]) + + # Calculate distance change (TA * 554 meters) + curr_distance = curr['ta_value'] * config.GSM_TA_METERS_PER_UNIT + prev_distance = prev['ta_value'] * config.GSM_TA_METERS_PER_UNIT + distance_change = abs(curr_distance - prev_distance) + + # Calculate time difference + curr_time = datetime.fromisoformat(curr['timestamp']) + prev_time = datetime.fromisoformat(prev['timestamp']) + time_diff_seconds = (curr_time - prev_time).total_seconds() + + # Calculate velocity (m/s) + if time_diff_seconds > 0: + velocity = distance_change / time_diff_seconds + else: + velocity = 0 + + # Store in velocity log + conn.execute(''' + INSERT INTO gsm_velocity_log + (device_id, prev_ta, curr_ta, prev_cid, curr_cid, estimated_velocity) + VALUES (?, ?, ?, ?, ?, ?) + ''', (device_id, prev['ta_value'], curr['ta_value'], + prev['cid'], curr['cid'], velocity)) + conn.commit() + + return jsonify({ + 'device_id': device_id, + 'velocity_mps': round(velocity, 2), + 'velocity_kmh': round(velocity * 3.6, 2), + 'distance_change_m': round(distance_change, 2), + 'time_diff_s': round(time_diff_seconds, 2) + }) + + except Exception as e: + logger.error(f"Error calculating velocity: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/crowd_density', methods=['GET']) +def get_crowd_density(): + """Get crowd density data by sector (G-09).""" + try: + hours = int(request.args.get('hours', 1)) # Last 1 hour by default + cid = request.args.get('cid') # Optional: specific cell + + with get_db() as conn: + # Count unique TMSI per cell in time window + query = ''' + SELECT + cid, + lac, + COUNT(DISTINCT tmsi) as unique_devices, + COUNT(*) as total_pings, + MIN(timestamp) as first_seen, + MAX(timestamp) as last_seen + FROM gsm_tmsi_log + WHERE timestamp >= datetime('now', '-' || ? || ' hours') + ''' + params = [hours] + + if cid: + query += ' AND cid = ?' + params.append(cid) + + query += ' GROUP BY cid, lac ORDER BY unique_devices DESC' + + results = conn.execute(query, params).fetchall() + density_data = [] + + for row in results: + density_data.append({ + 'cid': row['cid'], + 'lac': row['lac'], + 'unique_devices': row['unique_devices'], + 'total_pings': row['total_pings'], + 'first_seen': row['first_seen'], + 'last_seen': row['last_seen'], + 'density_level': 'high' if row['unique_devices'] > 20 else + 'medium' if row['unique_devices'] > 10 else 'low' + }) + + return jsonify(density_data) + + except Exception as e: + logger.error(f"Error fetching crowd density: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/life_patterns', methods=['GET']) +def get_life_patterns(): + """Get life pattern analysis for a device (G-10).""" + try: + device_id = request.args.get('device_id') + if not device_id: + # Return empty results gracefully when no device selected + return jsonify({ + 'device_id': None, + 'patterns': [], + 'message': 'No device selected' + }), 200 + + with get_db() as conn: + # Get historical signal data + results = conn.execute(''' + SELECT + strftime('%H', timestamp) as hour, + strftime('%w', timestamp) as day_of_week, + cid, + lac, + COUNT(*) as occurrences + FROM gsm_signals + WHERE (imsi = ? OR tmsi = ?) + AND timestamp >= datetime('now', '-60 days') + GROUP BY hour, day_of_week, cid, lac + ORDER BY occurrences DESC + ''', (device_id, device_id)).fetchall() + + patterns = [] + for row in results: + patterns.append({ + 'hour': int(row['hour']), + 'day_of_week': int(row['day_of_week']), + 'cid': row['cid'], + 'lac': row['lac'], + 'occurrences': row['occurrences'], + 'day_name': ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'][int(row['day_of_week'])] + }) + + # Identify regular patterns + regular_locations = [] + for pattern in patterns[:5]: # Top 5 most frequent + if pattern['occurrences'] >= 3: # Seen at least 3 times + regular_locations.append({ + 'cid': pattern['cid'], + 'typical_time': f"{pattern['day_name']} {pattern['hour']:02d}:00", + 'frequency': pattern['occurrences'] + }) + + return jsonify({ + 'device_id': device_id, + 'patterns': patterns, + 'regular_locations': regular_locations, + 'total_observations': sum(p['occurrences'] for p in patterns) + }) + + except Exception as e: + logger.error(f"Error analyzing life patterns: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/neighbor_audit', methods=['GET']) +def neighbor_audit(): + """Audit neighbor cell lists for consistency (G-11).""" + try: + cid = request.args.get('cid') + if not cid: + # Return empty results gracefully when no tower selected + return jsonify({ + 'cid': None, + 'neighbors': [], + 'inconsistencies': [], + 'message': 'No tower selected' + }), 200 + + with get_db() as conn: + # Get tower info with metadata (neighbor list stored in metadata JSON) + result = conn.execute(''' + SELECT metadata FROM gsm_cells WHERE cid = ? + ''', (cid,)).fetchone() + + if not result or not result['metadata']: + return jsonify({ + 'cid': cid, + 'status': 'no_data', + 'message': 'No neighbor list data available' + }) + + # Parse metadata JSON + metadata = json.loads(result['metadata']) + neighbor_list = metadata.get('neighbors', []) + + # Audit consistency + issues = [] + for neighbor_cid in neighbor_list: + # Check if neighbor exists in database + neighbor_exists = conn.execute(''' + SELECT id FROM gsm_cells WHERE cid = ? + ''', (neighbor_cid,)).fetchone() + + if not neighbor_exists: + issues.append({ + 'type': 'missing_neighbor', + 'cid': neighbor_cid, + 'message': f'Neighbor CID {neighbor_cid} not found in database' + }) + + return jsonify({ + 'cid': cid, + 'neighbor_count': len(neighbor_list), + 'neighbors': neighbor_list, + 'issues': issues, + 'status': 'suspicious' if issues else 'normal' + }) + + except Exception as e: + logger.error(f"Error auditing neighbors: {e}") + return jsonify({'error': str(e)}), 500 + + +@gsm_spy_bp.route('/traffic_correlation', methods=['GET']) +def traffic_correlation(): + """Correlate uplink/downlink traffic for pairing analysis (G-12).""" + try: + cid = request.args.get('cid') + minutes = int(request.args.get('minutes', 5)) + + with get_db() as conn: + # Get recent signal activity for this cell + results = conn.execute(''' + SELECT + imsi, + tmsi, + ta_value, + timestamp, + metadata + FROM gsm_signals + WHERE cid = ? + AND timestamp >= datetime('now', '-' || ? || ' minutes') + ORDER BY timestamp DESC + ''', (cid, minutes)).fetchall() + + correlations = [] + seen_devices = set() + + for row in results: + device_id = row['imsi'] or row['tmsi'] + if device_id and device_id not in seen_devices: + seen_devices.add(device_id) + + # Simple correlation: count bursts + burst_count = conn.execute(''' + SELECT COUNT(*) as bursts + FROM gsm_signals + WHERE (imsi = ? OR tmsi = ?) + AND cid = ? + AND timestamp >= datetime('now', '-' || ? || ' minutes') + ''', (device_id, device_id, cid, minutes)).fetchone() + + correlations.append({ + 'device_id': device_id, + 'burst_count': burst_count['bursts'], + 'last_seen': row['timestamp'], + 'ta_value': row['ta_value'], + 'activity_level': 'high' if burst_count['bursts'] > 10 else + 'medium' if burst_count['bursts'] > 5 else 'low' + }) + + return jsonify({ + 'cid': cid, + 'time_window_minutes': minutes, + 'active_devices': len(correlations), + 'correlations': correlations + }) + + except Exception as e: + logger.error(f"Error correlating traffic: {e}") + return jsonify({'error': str(e)}), 500 + + +# ============================================ +# Helper Functions +# ============================================ + +def parse_grgsm_scanner_output(line: str) -> dict[str, Any] | None: + """Parse grgsm_scanner output line. + + Actual output format is a table: + ARFCN | Freq (MHz) | CID | LAC | MCC | MNC | Power (dB) + -------------------------------------------------------------------- + 23 | 940.6 | 31245 | 1234 | 214 | 01 | -48 + """ + try: + # Skip progress, header, and separator lines + if 'Scanning:' in line or 'ARFCN' in line or '---' in line or 'Found' in line: + return None + + # Parse table row: " 23 | 940.6 | 31245 | 1234 | 214 | 01 | -48" + # Split by pipe and clean whitespace + parts = [p.strip() for p in line.split('|')] + + if len(parts) >= 7: + arfcn = parts[0] + freq = parts[1] + cid = parts[2] + lac = parts[3] + mcc = parts[4] + mnc = parts[5] + power = parts[6] + + # Validate that we have numeric data (not header line) + if arfcn.isdigit(): + data = { + 'type': 'tower', + 'arfcn': int(arfcn), + 'frequency': float(freq), + 'cid': int(cid), + 'lac': int(lac), + 'mcc': int(mcc), + 'mnc': int(mnc), + 'signal_strength': float(power), + 'timestamp': datetime.now().isoformat() + } + return data + + except Exception as e: + logger.debug(f"Failed to parse scanner line: {line} - {e}") + + return None + + +def parse_tshark_output(line: str) -> dict[str, Any] | None: + """Parse tshark filtered GSM output.""" + try: + # tshark output format: ta_value\ttmsi\timsi\tlac\tcid + parts = line.strip().split('\t') + + if len(parts) >= 5: + data = { + 'type': 'device', + 'ta_value': int(parts[0]) if parts[0] else None, + 'tmsi': parts[1] if parts[1] else None, + 'imsi': parts[2] if parts[2] else None, + 'lac': int(parts[3]) if parts[3] else None, + 'cid': int(parts[4]) if parts[4] else None, + 'timestamp': datetime.now().isoformat() + } + + # Calculate distance from TA + if data['ta_value'] is not None: + data['distance_meters'] = data['ta_value'] * config.GSM_TA_METERS_PER_UNIT + + return data + + except Exception as e: + logger.debug(f"Failed to parse tshark line: {line} - {e}") + + return None + + +def auto_start_monitor(tower_data): + """Automatically start monitoring the strongest tower found.""" + try: + arfcn = tower_data.get('arfcn') + if not arfcn: + logger.warning("Cannot auto-monitor: no ARFCN in tower data") + return + + logger.info(f"Auto-monitoring strongest tower: ARFCN {arfcn}, Signal {tower_data.get('signal_strength')} dBm") + + # Brief delay to ensure scanner has stabilized + time.sleep(2) + + with app_module.gsm_spy_lock: + if app_module.gsm_spy_monitor_process: + logger.info("Monitor already running, skipping auto-start") + return + + device_index = app_module.gsm_spy_active_device or 0 + + # Start and register monitoring (shared logic) + _start_and_register_monitor(arfcn, device_index) + + # Send SSE notification + try: + app_module.gsm_spy_queue.put_nowait({ + 'type': 'auto_monitor_started', + 'arfcn': arfcn, + 'tower': tower_data + }) + except queue.Full: + pass + + logger.info(f"Auto-monitoring started for ARFCN {arfcn}") + + except Exception as e: + logger.error(f"Error in auto-monitoring: {e}") + + +def scanner_thread(cmd, device_index): + """Thread to continuously run grgsm_scanner in a loop with non-blocking I/O. + + grgsm_scanner scans once and exits, so we loop it to provide + continuous updates to the dashboard. + """ + global gsm_towers_found + + strongest_tower = None + auto_monitor_triggered = False # Moved outside loop - persists across scans + scan_count = 0 + process = None + + try: + while app_module.gsm_spy_scanner_running: # Flag check + scan_count += 1 + logger.info(f"Starting GSM scan #{scan_count}") + + try: + # Start scanner process + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + bufsize=1 + ) + register_process(process) + logger.info(f"Started grgsm_scanner (PID: {process.pid})") + + # Standard pattern: reader threads with queue + output_queue_local = queue.Queue() + + def read_stdout(): + try: + for line in iter(process.stdout.readline, ''): + if line: + output_queue_local.put(('stdout', line)) + except Exception as e: + logger.error(f"stdout read error: {e}") + finally: + output_queue_local.put(('eof', None)) + + def read_stderr(): + try: + for line in iter(process.stderr.readline, ''): + if line: + logger.debug(f"grgsm_scanner: {line.strip()}") + except Exception as e: + logger.error(f"stderr read error: {e}") + + stdout_thread = threading.Thread(target=read_stdout, daemon=True) + stderr_thread = threading.Thread(target=read_stderr, daemon=True) + stdout_thread.start() + stderr_thread.start() + + # Process output with timeout + last_output = time.time() + scan_timeout = 120 # 2 minute maximum per scan + + while app_module.gsm_spy_scanner_running: + # Check if process died + if process.poll() is not None: + logger.info(f"Scanner exited (code: {process.returncode})") + break + + # Get output from queue with timeout + try: + msg_type, line = output_queue_local.get(timeout=1.0) + + if msg_type == 'eof': + break # EOF + + last_output = time.time() + + parsed = parse_grgsm_scanner_output(line) + if parsed: + # Enrich with coordinates + from utils.gsm_geocoding import enrich_tower_data + enriched = enrich_tower_data(parsed) + + # Store in DataStore + key = f"{enriched['mcc']}_{enriched['mnc']}_{enriched['lac']}_{enriched['cid']}" + app_module.gsm_spy_towers[key] = enriched + + # Track strongest tower + signal = enriched.get('signal_strength', -999) + if strongest_tower is None or signal > strongest_tower.get('signal_strength', -999): + strongest_tower = enriched + + # Queue for SSE + try: + app_module.gsm_spy_queue.put_nowait(enriched) + except queue.Full: + logger.warning("Queue full, dropping tower update") + + # Thread-safe counter update + with app_module.gsm_spy_lock: + gsm_towers_found += 1 + current_count = gsm_towers_found + + # Auto-monitor strongest tower (once per session) + if current_count >= 3 and not auto_monitor_triggered and strongest_tower: + auto_monitor_triggered = True + logger.info("Auto-starting monitor on strongest tower") + threading.Thread( + target=auto_start_monitor, + args=(strongest_tower,), + daemon=True + ).start() + except queue.Empty: + # No output, check timeout + if time.time() - last_output > scan_timeout: + logger.warning(f"Scan timeout after {scan_timeout}s") + break + + # Clean up process with timeout + if process.poll() is None: + logger.info("Terminating scanner process") + safe_terminate(process, timeout=5) + else: + process.wait() # Reap zombie + + logger.info(f"Scan #{scan_count} complete") + + except Exception as e: + logger.error(f"Scanner scan error: {e}", exc_info=True) + if process and process.poll() is None: + safe_terminate(process) + + # Check if should continue + if not app_module.gsm_spy_scanner_running: + break + + # Wait between scans with responsive flag checking + logger.info("Waiting 5 seconds before next scan") + for i in range(5): + if not app_module.gsm_spy_scanner_running: + break + time.sleep(1) + + except Exception as e: + logger.error(f"Scanner thread fatal error: {e}", exc_info=True) + + finally: + # Always cleanup + if process and process.poll() is None: + safe_terminate(process, timeout=5) + + logger.info("Scanner thread terminated") + + # Reset global state + with app_module.gsm_spy_lock: + app_module.gsm_spy_scanner_running = False + if app_module.gsm_spy_active_device is not None: + from app import release_sdr_device + release_sdr_device(app_module.gsm_spy_active_device) + app_module.gsm_spy_active_device = None + + +def monitor_thread(process): + """Thread to read tshark output using standard iter pattern.""" + global gsm_devices_tracked + + # Standard pattern: reader thread with queue + output_queue_local = queue.Queue() + + def read_stdout(): + try: + for line in iter(process.stdout.readline, ''): + if line: + output_queue_local.put(('stdout', line)) + except Exception as e: + logger.error(f"tshark read error: {e}") + finally: + output_queue_local.put(('eof', None)) + + stdout_thread = threading.Thread(target=read_stdout, daemon=True) + stdout_thread.start() + + try: + while app_module.gsm_spy_monitor_process: + # Check if process died + if process.poll() is not None: + logger.info(f"Monitor process exited (code: {process.returncode})") + break + + # Get output from queue with timeout + try: + msg_type, line = output_queue_local.get(timeout=1.0) + except queue.Empty: + continue # Timeout, check flag again + + if msg_type == 'eof': + break # EOF + + parsed = parse_tshark_output(line) + if parsed: + # Store in DataStore + key = parsed.get('tmsi') or parsed.get('imsi') or str(time.time()) + app_module.gsm_spy_devices[key] = parsed + + # Queue for SSE stream + try: + app_module.gsm_spy_queue.put_nowait(parsed) + except queue.Full: + pass + + # Store in database for historical analysis + try: + with get_db() as conn: + # gsm_signals table + conn.execute(''' + INSERT INTO gsm_signals + (imsi, tmsi, lac, cid, ta_value, arfcn) + VALUES (?, ?, ?, ?, ?, ?) + ''', ( + parsed.get('imsi'), + parsed.get('tmsi'), + parsed.get('lac'), + parsed.get('cid'), + parsed.get('ta_value'), + app_module.gsm_spy_selected_arfcn + )) + + # gsm_tmsi_log table for crowd density + if parsed.get('tmsi'): + conn.execute(''' + INSERT INTO gsm_tmsi_log + (tmsi, lac, cid, ta_value) + VALUES (?, ?, ?, ?) + ''', ( + parsed.get('tmsi'), + parsed.get('lac'), + parsed.get('cid'), + parsed.get('ta_value') + )) + + # Velocity calculation (G-08) + device_id = parsed.get('imsi') or parsed.get('tmsi') + if device_id and parsed.get('ta_value') is not None: + # Get previous TA reading + prev_reading = conn.execute(''' + SELECT ta_value, cid, timestamp + FROM gsm_signals + WHERE (imsi = ? OR tmsi = ?) + ORDER BY timestamp DESC + LIMIT 1 OFFSET 1 + ''', (device_id, device_id)).fetchone() + + if prev_reading: + # Calculate velocity + curr_ta = parsed.get('ta_value') + prev_ta = prev_reading['ta_value'] + curr_distance = curr_ta * config.GSM_TA_METERS_PER_UNIT + prev_distance = prev_ta * config.GSM_TA_METERS_PER_UNIT + distance_change = abs(curr_distance - prev_distance) + + # Time difference + prev_time = datetime.fromisoformat(prev_reading['timestamp']) + curr_time = datetime.now() + time_diff_seconds = (curr_time - prev_time).total_seconds() + + if time_diff_seconds > 0: + velocity = distance_change / time_diff_seconds + + # Store velocity + conn.execute(''' + INSERT INTO gsm_velocity_log + (device_id, prev_ta, curr_ta, prev_cid, curr_cid, estimated_velocity) + VALUES (?, ?, ?, ?, ?, ?) + ''', ( + device_id, + prev_ta, + curr_ta, + prev_reading['cid'], + parsed.get('cid'), + velocity + )) + + conn.commit() + except Exception as e: + logger.error(f"Error storing device data: {e}") + + # Thread-safe counter + with app_module.gsm_spy_lock: + gsm_devices_tracked += 1 + + except Exception as e: + logger.error(f"Monitor thread error: {e}", exc_info=True) + + finally: + # Reap process with timeout + try: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.warning("Monitor process didn't terminate, killing") + process.kill() + process.wait() + else: + process.wait() + logger.info(f"Monitor process exited with code {process.returncode}") + except Exception as e: + logger.error(f"Error reaping monitor process: {e}") + + logger.info("Monitor thread terminated") diff --git a/setup.sh b/setup.sh index 7cdf6f1..9174906 100755 --- a/setup.sh +++ b/setup.sh @@ -694,6 +694,52 @@ install_macos_packages() { progress "Installing gpsd" brew_install gpsd + # gr-gsm for GSM Intelligence + if ! cmd_exists grgsm_scanner; then + echo + info "gr-gsm provides GSM cellular signal decoding..." + if ask_yes_no "Do you want to install gr-gsm?"; then + progress "Installing gr-gsm" + brew_install gnuradio + (brew_install gr-gsm) || { + warn "gr-gsm not available in Homebrew, attempting manual build..." + # Manual build instructions + if ask_yes_no "Attempt to build gr-gsm from source? (requires CMake and build tools)"; then + info "Cloning gr-gsm repository..." + git clone https://github.com/ptrkrysik/gr-gsm.git /tmp/gr-gsm + cd /tmp/gr-gsm + mkdir build && cd build + cmake .. + make -j$(sysctl -n hw.ncpu) + sudo make install + cd ~ + rm -rf /tmp/gr-gsm + ok "gr-gsm installed successfully" + else + warn "Skipping gr-gsm source build. GSM Spy feature will not work." + fi + } + else + warn "Skipping gr-gsm installation. GSM Spy feature will not work." + fi + else + ok "gr-gsm already installed" + fi + + # Wireshark (tshark) for packet analysis + if ! cmd_exists tshark; then + echo + info "tshark is used for GSM packet parsing..." + if ask_yes_no "Do you want to install tshark?"; then + progress "Installing Wireshark (tshark)" + brew_install wireshark + else + warn "Skipping tshark installation." + fi + else + ok "tshark already installed" + fi + progress "Installing Ubertooth tools (optional)" if ! cmd_exists ubertooth-btle; then echo @@ -1104,6 +1150,87 @@ install_debian_packages() { progress "Installing gpsd" apt_install gpsd gpsd-clients || true + # gr-gsm for GSM Intelligence + if ! cmd_exists grgsm_scanner; then + echo + info "gr-gsm provides GSM cellular signal decoding..." + if ask_yes_no "Do you want to install gr-gsm?"; then + progress "Installing GNU Radio and gr-gsm" + # Try to install gr-gsm directly from package repositories + apt_install gnuradio gnuradio-dev gr-osmosdr gr-gsm || { + warn "gr-gsm package not available in repositories. Attempting source build..." + + # Fallback: Build from source + progress "Building gr-gsm from source" + apt_install git cmake libboost-all-dev libcppunit-dev swig \ + doxygen liblog4cpp5-dev python3-scipy python3-numpy \ + libvolk-dev libuhd-dev libfftw3-dev || true + + info "Cloning gr-gsm repository..." + if [ -d /tmp/gr-gsm ]; then + rm -rf /tmp/gr-gsm + fi + + git clone https://github.com/ptrkrysik/gr-gsm.git /tmp/gr-gsm || { + warn "Failed to clone gr-gsm repository. GSM Spy will not be available." + return 0 + } + + cd /tmp/gr-gsm + mkdir -p build && cd build + + # Try to find GNU Radio cmake files + if [ -d /usr/lib/x86_64-linux-gnu/cmake/gnuradio ]; then + export CMAKE_PREFIX_PATH="/usr/lib/x86_64-linux-gnu/cmake/gnuradio:$CMAKE_PREFIX_PATH" + fi + + info "Running CMake configuration..." + if cmake .. 2>/dev/null; then + info "Compiling gr-gsm (this may take several minutes)..." + if make -j$(nproc) 2>/dev/null; then + $SUDO make install + $SUDO ldconfig + cd ~ + rm -rf /tmp/gr-gsm + ok "gr-gsm built and installed successfully" + else + warn "gr-gsm compilation failed. GSM Spy feature will not work." + cd ~ + rm -rf /tmp/gr-gsm + fi + else + warn "gr-gsm CMake configuration failed. GNU Radio 3.8+ may not be available." + cd ~ + rm -rf /tmp/gr-gsm + fi + } + + # Verify installation + if cmd_exists grgsm_scanner; then + ok "gr-gsm installed successfully" + else + warn "gr-gsm installation incomplete. GSM Spy feature will not work." + fi + else + warn "Skipping gr-gsm installation." + fi + else + ok "gr-gsm already installed" + fi + + # Wireshark (tshark) + if ! cmd_exists tshark; then + echo + info "Installing tshark for GSM packet analysis..." + apt_install tshark || true + # Allow non-root capture + $SUDO dpkg-reconfigure wireshark-common 2>/dev/null || true + $SUDO usermod -a -G wireshark $USER 2>/dev/null || true + ok "tshark installed. You may need to re-login for wireshark group permissions." + else + ok "tshark already installed" + fi + progress "Installing Python packages" apt_install python3-venv python3-pip || true # Install Python packages via apt (more reliable than pip on modern Debian/Ubuntu) diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html new file mode 100644 index 0000000..e2e6776 --- /dev/null +++ b/templates/gsm_spy_dashboard.html @@ -0,0 +1,2396 @@ + + + + + + GSM SPY // INTERCEPT - See the Invisible + + {% if offline_settings.fonts_source == 'local' %} + + {% else %} + + {% endif %} + + {% if offline_settings.assets_source == 'local' %} + + + {% else %} + + + {% endif %} + + + + + + + + + + + +
+
+ +
+ +
+
+ STANDBY +
+
+ + {% set active_mode = 'gsm' %} + {% include 'partials/nav.html' with context %} + + +
+
+
+ 0 + TOWERS +
+
+ 0 + DEVICES +
+
+ 0 + ROGUES +
+
+ 0 + SIGNALS +
+
+ - + CROWD +
+
+
+
+ STANDBY +
+
--:--:-- UTC
+ +
+
+ + +
+
+
+
Analytics Overview
+ +
+
+
+ +
+
+
📍
+
Velocity Tracking
+
+
+ Track device movement by analyzing Timing Advance transitions and cell handovers. + Estimates velocity and direction based on TA delta and cell sector patterns. +
+
+
+
0
+
Devices Tracked
+
+
+
- km/h
+
Avg Velocity
+
+
+
+ + +
+
+
👥
+
Crowd Density
+
+
+ Aggregate TMSI pings per cell sector to estimate crowd density. + Visualizes hotspots and congestion patterns across towers. +
+
+
+
0
+
Total Devices
+
+
+
0
+
Peak Sector
+
+
+
+ + +
+
+
📊
+
Life Patterns
+
+
+ Analyze 60-day historical data to identify recurring patterns in device behavior. + Detects work locations, commute routes, and daily routines. +
+
+
+
0
+
Patterns Found
+
+
+
0%
+
Confidence
+
+
+
+ + +
+
+
🔍
+
Neighbor Audit
+
+
+ Validate neighbor cell lists against expected network topology. + Detects inconsistencies that may indicate rogue towers. +
+
+
+
0
+
Neighbors
+
+
+
0
+
Anomalies
+
+
+
+ + +
+
+
📡
+
Traffic Correlation
+
+
+ Correlate uplink and downlink timing to identify communication patterns. + Maps device-to-device interactions and network flows. +
+
+
+
0
+
Paired Flows
+
+
+
0
+
Active Now
+
+
+
+
+
+
+
+ +
+ +
+ +
+
+ SIGNAL SOURCE +
+
+
+
+ + +
+
+
+ + +
+
+ REGION / BANDS +
+
+
+ + + +
+
+ + +
+
+ ADVANCED ANALYSIS +
+
+
+ + + + + +
+ +
+
+ + +
+
+
+ + +
+ +
+
+ SELECTED TOWER +
+
+
+
+
📡
+
Click a tower on the map
+
+
+
+ + +
+
+ TRACKED TOWERS +
+
+
+
+
No towers detected
+
Start scanner to begin
+
+
+
+ + +
+
+ TRACKED DEVICES +
+
+
+
+
No devices tracked
+
Devices will appear here
+
+
+
+ + +
+
+ ALERTS +
+
+
+
+
No alerts
+
+
+
+
+ + +
+ +
+ GPS LOCATION +
+ + + +
+
+ + +
+ GSM SCANNER +
+ + + +
+
+
+
+ + + + + + + diff --git a/templates/index.html b/templates/index.html index 7d635ef..373005e 100644 --- a/templates/index.html +++ b/templates/index.html @@ -171,6 +171,10 @@ Vessels + + + GSM SPY +