From c2891938ab33bfdae4799eef7d9b23bf92fd1590 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 22:04:12 +0000 Subject: [PATCH] Remove GSM spy functionality for legal compliance Remove all GSM cellular intelligence features including tower scanning, signal monitoring, rogue detection, crowd density analysis, and OpenCellID integration across routes, templates, utils, tests, and build configuration. Co-Authored-By: Claude Opus 4.6 --- Dockerfile | 28 - app.py | 59 +- config.py | 31 +- routes/__init__.py | 2 - routes/gsm_spy.py | 2051 --------------- setup.sh | 127 +- static/js/core/settings-manager.js | 55 +- templates/gsm_spy_dashboard.html | 3300 ------------------------ templates/index.html | 4 - templates/partials/nav.html | 1 - templates/partials/settings-modal.html | 65 - tests/test_gsm_spy.py | 360 --- utils/constants.py | 10 - utils/database.py | 185 -- utils/dependencies.py | 32 - utils/gsm_geocoding.py | 226 -- utils/logging.py | 1 - 17 files changed, 21 insertions(+), 6516 deletions(-) delete mode 100644 routes/gsm_spy.py delete mode 100644 templates/gsm_spy_dashboard.html delete mode 100644 tests/test_gsm_spy.py delete mode 100644 utils/gsm_geocoding.py diff --git a/Dockerfile b/Dockerfile index a435d81..0b45b00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,39 +48,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ airspy \ limesuite \ hackrf \ - # GSM Intelligence (tshark for packet parsing) - tshark \ # Utilities curl \ procps \ && rm -rf /var/lib/apt/lists/* -# GSM Intelligence: gr-gsm (grgsm_scanner, grgsm_livemon) -# Install from apt if available, otherwise build from source -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - gnuradio gr-osmosdr gr-gsm 2>/dev/null \ - || ( \ - apt-get install -y --no-install-recommends \ - gnuradio gnuradio-dev gr-osmosdr \ - git cmake libboost-all-dev libcppunit-dev swig \ - doxygen liblog4cpp5-dev python3-scipy python3-numpy \ - libvolk-dev libfftw3-dev build-essential \ - && cd /tmp \ - && git clone --depth 1 https://github.com/bkerler/gr-gsm.git \ - && cd gr-gsm \ - && mkdir build && cd build \ - && cmake .. \ - && make -j$(nproc) \ - && make install \ - && ldconfig \ - && rm -rf /tmp/gr-gsm \ - && apt-get remove -y gnuradio-dev libcppunit-dev swig doxygen \ - liblog4cpp5-dev libvolk-dev build-essential git cmake \ - && apt-get autoremove -y \ - ) \ - && rm -rf /var/lib/apt/lists/* - # Build dump1090-fa and acarsdec from source (packages not available in slim repos) RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ diff --git a/app.py b/app.py index 76ee903..bb91402 100644 --- a/app.py +++ b/app.py @@ -39,7 +39,6 @@ from utils.constants import ( MAX_VESSEL_AGE_SECONDS, MAX_DSC_MESSAGE_AGE_SECONDS, MAX_DEAUTH_ALERTS_AGE_SECONDS, - MAX_GSM_AGE_SECONDS, QUEUE_MAX_SIZE, ) import logging @@ -188,16 +187,6 @@ deauth_detector = None deauth_detector_queue = queue.Queue(maxsize=QUEUE_MAX_SIZE) deauth_detector_lock = threading.Lock() -# GSM Spy -gsm_spy_scanner_running = False # Flag: scanner thread active -gsm_spy_livemon_process = None # For grgsm_livemon process -gsm_spy_monitor_process = None # For tshark monitoring process -gsm_spy_queue = queue.Queue(maxsize=QUEUE_MAX_SIZE) -gsm_spy_lock = threading.Lock() -gsm_spy_active_device = None -gsm_spy_selected_arfcn = None -gsm_spy_region = 'Americas' # Default band - # ============================================ # GLOBAL STATE DICTIONARIES # ============================================ @@ -230,16 +219,6 @@ dsc_messages = DataStore(max_age_seconds=MAX_DSC_MESSAGE_AGE_SECONDS, name='dsc_ # Deauth alerts - using DataStore for automatic cleanup deauth_alerts = DataStore(max_age_seconds=MAX_DEAUTH_ALERTS_AGE_SECONDS, name='deauth_alerts') -# GSM Spy data stores -gsm_spy_towers = DataStore( - max_age_seconds=MAX_GSM_AGE_SECONDS, - name='gsm_spy_towers' -) -gsm_spy_devices = DataStore( - max_age_seconds=MAX_GSM_AGE_SECONDS, - name='gsm_spy_devices' -) - # Satellite state satellite_passes = [] # Predicted satellite passes (not auto-cleaned, calculated) @@ -252,8 +231,6 @@ cleanup_manager.register(adsb_aircraft) cleanup_manager.register(ais_vessels) cleanup_manager.register(dsc_messages) cleanup_manager.register(deauth_alerts) -cleanup_manager.register(gsm_spy_towers) -cleanup_manager.register(gsm_spy_devices) # ============================================ # SDR DEVICE REGISTRY @@ -687,8 +664,6 @@ def kill_all() -> Response: global current_process, sensor_process, wifi_process, adsb_process, ais_process, acars_process global aprs_process, aprs_rtl_process, dsc_process, dsc_rtl_process, bt_process global dmr_process, dmr_rtl_process - global gsm_spy_livemon_process, gsm_spy_monitor_process - global gsm_spy_scanner_running, gsm_spy_active_device, gsm_spy_selected_arfcn, gsm_spy_region # Import adsb and ais modules to reset their state from routes import adsb as adsb_module @@ -701,8 +676,7 @@ def kill_all() -> Response: 'airodump-ng', 'aireplay-ng', 'airmon-ng', 'dump1090', 'acarsdec', 'direwolf', 'AIS-catcher', 'hcitool', 'bluetoothctl', 'dsd', - 'rtl_tcp', 'rtl_power', 'rtlamr', 'ffmpeg', - 'grgsm_scanner', 'grgsm_livemon', 'tshark' + 'rtl_tcp', 'rtl_power', 'rtlamr', 'ffmpeg' ] for proc in processes_to_kill: @@ -771,29 +745,6 @@ def kill_all() -> Response: except Exception: pass - # Reset GSM Spy state - with gsm_spy_lock: - gsm_spy_scanner_running = False - gsm_spy_active_device = None - gsm_spy_selected_arfcn = None - gsm_spy_region = 'Americas' - - if gsm_spy_livemon_process: - try: - if safe_terminate(gsm_spy_livemon_process): - killed.append('grgsm_livemon') - except Exception: - pass - gsm_spy_livemon_process = None - - if gsm_spy_monitor_process: - try: - if safe_terminate(gsm_spy_monitor_process): - killed.append('tshark') - except Exception: - pass - gsm_spy_monitor_process = None - # Clear SDR device registry with sdr_device_registry_lock: sdr_device_registry.clear() @@ -885,19 +836,11 @@ def main() -> None: # Register database cleanup functions from utils.database import ( - cleanup_old_gsm_signals, - cleanup_old_gsm_tmsi_log, - cleanup_old_gsm_velocity_log, cleanup_old_signal_history, cleanup_old_timeline_entries, cleanup_old_dsc_alerts, cleanup_old_payloads ) - # GSM cleanups: signals (60 days), TMSI log (24 hours), velocity (1 hour) - # Interval multiplier: cleanup every N cycles (60s interval = 1 cleanup per hour at multiplier 60) - cleanup_manager.register_db_cleanup(cleanup_old_gsm_tmsi_log, interval_multiplier=60) # Every hour - cleanup_manager.register_db_cleanup(cleanup_old_gsm_velocity_log, interval_multiplier=60) # Every hour - cleanup_manager.register_db_cleanup(cleanup_old_gsm_signals, interval_multiplier=1440) # Every 24 hours cleanup_manager.register_db_cleanup(cleanup_old_signal_history, interval_multiplier=1440) # Every 24 hours cleanup_manager.register_db_cleanup(cleanup_old_timeline_entries, interval_multiplier=1440) # Every 24 hours cleanup_manager.register_db_cleanup(cleanup_old_dsc_alerts, interval_multiplier=1440) # Every 24 hours diff --git a/config.py b/config.py index 98f2d1d..2e01d48 100644 --- a/config.py +++ b/config.py @@ -204,25 +204,20 @@ SATELLITE_UPDATE_INTERVAL = _get_env_int('SATELLITE_UPDATE_INTERVAL', 30) SATELLITE_TRAJECTORY_POINTS = _get_env_int('SATELLITE_TRAJECTORY_POINTS', 30) SATELLITE_ORBIT_MINUTES = _get_env_int('SATELLITE_ORBIT_MINUTES', 45) -# Update checking -GITHUB_REPO = _get_env('GITHUB_REPO', 'smittix/intercept') -UPDATE_CHECK_ENABLED = _get_env_bool('UPDATE_CHECK_ENABLED', True) -UPDATE_CHECK_INTERVAL_HOURS = _get_env_int('UPDATE_CHECK_INTERVAL_HOURS', 6) - -# Alerting -ALERT_WEBHOOK_URL = _get_env('ALERT_WEBHOOK_URL', '') -ALERT_WEBHOOK_SECRET = _get_env('ALERT_WEBHOOK_SECRET', '') -ALERT_WEBHOOK_TIMEOUT = _get_env_int('ALERT_WEBHOOK_TIMEOUT', 5) - -# Admin credentials -ADMIN_USERNAME = _get_env('ADMIN_USERNAME', 'admin') -ADMIN_PASSWORD = _get_env('ADMIN_PASSWORD', 'admin') +# Update checking +GITHUB_REPO = _get_env('GITHUB_REPO', 'smittix/intercept') +UPDATE_CHECK_ENABLED = _get_env_bool('UPDATE_CHECK_ENABLED', True) +UPDATE_CHECK_INTERVAL_HOURS = _get_env_int('UPDATE_CHECK_INTERVAL_HOURS', 6) + +# Alerting +ALERT_WEBHOOK_URL = _get_env('ALERT_WEBHOOK_URL', '') +ALERT_WEBHOOK_SECRET = _get_env('ALERT_WEBHOOK_SECRET', '') +ALERT_WEBHOOK_TIMEOUT = _get_env_int('ALERT_WEBHOOK_TIMEOUT', 5) + +# Admin credentials +ADMIN_USERNAME = _get_env('ADMIN_USERNAME', 'admin') +ADMIN_PASSWORD = _get_env('ADMIN_PASSWORD', 'admin') -# GSM Spy settings -GSM_OPENCELLID_API_KEY = _get_env('GSM_OPENCELLID_API_KEY', '') -GSM_OPENCELLID_API_URL = _get_env('GSM_OPENCELLID_API_URL', 'https://opencellid.org/cell/get') -GSM_API_DAILY_LIMIT = _get_env_int('GSM_API_DAILY_LIMIT', 1000) -GSM_TA_METERS_PER_UNIT = _get_env_int('GSM_TA_METERS_PER_UNIT', 554) def configure_logging() -> None: """Configure application logging.""" diff --git a/routes/__init__.py b/routes/__init__.py index af211cc..0ac1fc2 100644 --- a/routes/__init__.py +++ b/routes/__init__.py @@ -31,7 +31,6 @@ def register_blueprints(app): from .websdr import websdr_bp from .alerts import alerts_bp from .recordings import recordings_bp - from .gsm_spy import gsm_spy_bp app.register_blueprint(pager_bp) app.register_blueprint(sensor_bp) @@ -62,7 +61,6 @@ def register_blueprints(app): app.register_blueprint(websdr_bp) # HF/Shortwave WebSDR app.register_blueprint(alerts_bp) # Cross-mode alerts app.register_blueprint(recordings_bp) # Session recordings - app.register_blueprint(gsm_spy_bp) # GSM cellular intelligence # Initialize TSCM state with queue and lock from app import app as app_module diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py deleted file mode 100644 index 49aca53..0000000 --- a/routes/gsm_spy.py +++ /dev/null @@ -1,2051 +0,0 @@ -"""GSM Spy route handlers for cellular tower and device tracking.""" - -from __future__ import annotations - -import json -import logging -import os -import queue -import re -import shutil -import subprocess -import threading -import time -from datetime import datetime, timedelta -from typing import Any - -import requests -from flask import Blueprint, Response, jsonify, render_template, request - -import app as app_module -import config -from config import SHARED_OBSERVER_LOCATION_ENABLED -from utils.database import get_db -from utils.process import register_process, safe_terminate, unregister_process -from utils.sse import format_sse -from utils.validation import validate_device_index - -from utils.logging import get_logger -logger = get_logger('intercept.gsm_spy') -logger.setLevel(logging.DEBUG) # GSM Spy needs verbose logging for diagnostics - -gsm_spy_bp = Blueprint('gsm_spy', __name__, url_prefix='/gsm_spy') - -# Regional band configurations (G-01) -REGIONAL_BANDS = { - 'Americas': { - 'GSM850': {'start': 869e6, 'end': 894e6, 'arfcn_start': 128, 'arfcn_end': 251}, - 'PCS1900': {'start': 1930e6, 'end': 1990e6, 'arfcn_start': 512, 'arfcn_end': 810} - }, - 'Europe': { - 'GSM800': {'start': 832e6, 'end': 862e6, 'arfcn_start': 438, 'arfcn_end': 511}, # E-GSM800 downlink - 'GSM850': {'start': 869e6, 'end': 894e6, 'arfcn_start': 128, 'arfcn_end': 251}, # Also used in some EU countries - 'EGSM900': {'start': 935e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, # DL = 935 + 0.2*ARFCN - 'EGSM900_EXT': {'start': 925.2e6, 'end': 935e6, 'arfcn_start': 975, 'arfcn_end': 1023}, # E-GSM extension - 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} - }, - 'Asia': { - 'EGSM900': {'start': 935e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, # DL = 935 + 0.2*ARFCN - 'EGSM900_EXT': {'start': 925.2e6, 'end': 935e6, 'arfcn_start': 975, 'arfcn_end': 1023}, # E-GSM extension - 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} - } -} - -# Module state tracking -gsm_using_service = False -gsm_connected = False -gsm_towers_found = 0 -gsm_devices_tracked = 0 - -# Geocoding worker state -_geocoding_worker_thread = None - - -# ============================================ -# API Usage Tracking Helper Functions -# ============================================ - -def get_api_usage_today(): - """Get OpenCellID API usage count for today.""" - from utils.database import get_setting - today = datetime.now().date().isoformat() - usage_date = get_setting('gsm.opencellid.usage_date', '') - - # Reset counter if new day - if usage_date != today: - from utils.database import set_setting - set_setting('gsm.opencellid.usage_date', today) - set_setting('gsm.opencellid.usage_count', 0) - return 0 - - return get_setting('gsm.opencellid.usage_count', 0) - - -def increment_api_usage(): - """Increment OpenCellID API usage counter.""" - from utils.database import set_setting - current = get_api_usage_today() - set_setting('gsm.opencellid.usage_count', current + 1) - return current + 1 - - -def get_opencellid_api_key(): - """Get OpenCellID API key, checking env var first, then database setting.""" - env_key = config.GSM_OPENCELLID_API_KEY - if env_key: - return env_key - from utils.database import get_setting - return get_setting('gsm.opencellid.api_key', '') - - -def can_use_api(): - """Check if we can make an API call within daily limit.""" - if not get_opencellid_api_key(): - return False - current_usage = get_api_usage_today() - return current_usage < config.GSM_API_DAILY_LIMIT - - -# ============================================ -# Background Geocoding Worker -# ============================================ - -def start_geocoding_worker(): - """Start background thread for async geocoding.""" - global _geocoding_worker_thread - - # Clean poisoned cache entries (rows with NULL lat/lon from failed API responses) - try: - with get_db() as conn: - deleted = conn.execute( - 'DELETE FROM gsm_cells WHERE lat IS NULL OR lon IS NULL' - ).rowcount - conn.commit() - if deleted: - logger.info(f"Cleaned {deleted} poisoned cache entries (NULL coordinates)") - except Exception as e: - logger.warning(f"Could not clean cache: {e}") - - if _geocoding_worker_thread is None or not _geocoding_worker_thread.is_alive(): - _geocoding_worker_thread = threading.Thread( - target=geocoding_worker, - daemon=True, - name='gsm-geocoding-worker' - ) - _geocoding_worker_thread.start() - logger.info("Started geocoding worker thread") - - -def geocoding_worker(): - """Worker thread processes pending geocoding requests.""" - from utils.gsm_geocoding import lookup_cell_from_api, get_geocoding_queue - - geocoding_queue = get_geocoding_queue() - - while True: - try: - # Wait for pending tower with timeout - tower_data = geocoding_queue.get(timeout=5) - - # Check API key and rate limit - if not can_use_api(): - if not get_opencellid_api_key(): - logger.warning("OpenCellID API key not configured (set INTERCEPT_GSM_OPENCELLID_API_KEY or configure in Settings > API Keys)") - else: - current_usage = get_api_usage_today() - logger.warning(f"OpenCellID API daily limit reached ({current_usage}/{config.GSM_API_DAILY_LIMIT})") - geocoding_queue.task_done() - continue - - # Call API - mcc = tower_data.get('mcc') - mnc = tower_data.get('mnc') - lac = tower_data.get('lac') - cid = tower_data.get('cid') - - logger.debug(f"Geocoding tower via API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - - coords = lookup_cell_from_api(mcc, mnc, lac, cid) - - if coords: - # Update tower data with coordinates - tower_data['lat'] = coords['lat'] - tower_data['lon'] = coords['lon'] - tower_data['source'] = 'api' - tower_data['status'] = 'resolved' - tower_data['type'] = 'tower_update' - - # Add optional fields if available - if coords.get('azimuth') is not None: - tower_data['azimuth'] = coords['azimuth'] - if coords.get('range_meters') is not None: - tower_data['range_meters'] = coords['range_meters'] - if coords.get('operator'): - tower_data['operator'] = coords['operator'] - if coords.get('radio'): - tower_data['radio'] = coords['radio'] - - # Update DataStore - key = f"{mcc}_{mnc}_{lac}_{cid}" - app_module.gsm_spy_towers[key] = tower_data - - # Send update to SSE stream - try: - app_module.gsm_spy_queue.put_nowait(tower_data) - logger.info(f"Resolved coordinates for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - except queue.Full: - logger.warning("SSE queue full, dropping tower update") - - # Increment API usage counter - usage_count = increment_api_usage() - logger.info(f"OpenCellID API call #{usage_count} today") - - else: - logger.warning(f"Could not resolve coordinates for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - - geocoding_queue.task_done() - - # Rate limiting between API calls (be nice to OpenCellID) - time.sleep(1) - - except queue.Empty: - # No pending towers, continue waiting - continue - except Exception as e: - logger.error(f"Geocoding worker error: {e}", exc_info=True) - time.sleep(1) - - -def arfcn_to_frequency(arfcn): - """Convert ARFCN to downlink frequency in Hz. - - Uses REGIONAL_BANDS to determine the correct band and conversion formula. - Returns frequency in Hz (e.g., 925800000 for 925.8 MHz). - """ - arfcn = int(arfcn) - - # Search all bands to find which one this ARFCN belongs to - for region_bands in REGIONAL_BANDS.values(): - for band_name, band_info in region_bands.items(): - arfcn_start = band_info['arfcn_start'] - arfcn_end = band_info['arfcn_end'] - - if arfcn_start <= arfcn <= arfcn_end: - # Found the right band, calculate frequency - # Downlink frequency = band_start + (arfcn - arfcn_start) * 200kHz - freq_hz = band_info['start'] + (arfcn - arfcn_start) * 200000 - return int(freq_hz) - - # If ARFCN not found in any band, raise error - raise ValueError(f"ARFCN {arfcn} not found in any known GSM band") - - -def validate_band_names(bands: list[str], region: str) -> tuple[list[str], str | None]: - """Validate band names against REGIONAL_BANDS whitelist. - - Args: - bands: List of band names from user input - region: Region name (Americas, Europe, Asia) - - Returns: - Tuple of (validated_bands, error_message) - """ - if not bands: - return [], None - - region_bands = REGIONAL_BANDS.get(region) - if not region_bands: - return [], f"Invalid region: {region}" - - valid_band_names = set(region_bands.keys()) - invalid_bands = [b for b in bands if b not in valid_band_names] - - if invalid_bands: - return [], (f"Invalid bands for {region}: {', '.join(invalid_bands)}. " - f"Valid bands: {', '.join(sorted(valid_band_names))}") - - return bands, None - - -# tshark field name discovery - field names vary between Wireshark versions -_tshark_fields_cache: dict[str, str] | None = None - - -def _discover_tshark_fields() -> dict[str, str | None]: - """Discover correct tshark field names for GSM A protocol. - - Searches tshark's registered fields for keywords to find the actual - names used by the installed Wireshark version, then validates them. - - Returns: - Dict mapping logical names to actual tshark field names (or None): - {'ta': ..., 'tmsi': ..., 'imsi': ..., 'lac': ..., 'cid': ...} - """ - global _tshark_fields_cache - if _tshark_fields_cache is not None: - return _tshark_fields_cache - - # Search patterns for each logical field (applied to tshark -G fields output) - # Each entry: (keyword_patterns, exclusion_patterns) - # We search the field filter name column for matches - search_config = { - 'ta': { - 'keywords': ['timing_adv', 'timing_advance'], - 'prefer': ['gsm_a'], # prefer GSM A protocol fields - }, - 'tmsi': { - 'keywords': ['.tmsi'], - 'prefer': ['gsm_a'], - }, - 'imsi': { - 'keywords': ['.imsi'], - 'prefer': ['e212', 'gsm_a'], - }, - 'lac': { - 'keywords': ['.lac'], - 'prefer': ['gsm_a', 'e212'], - }, - 'cid': { - 'keywords': ['cellid', 'cell_ci', '.cell_id', 'e212.ci'], - 'prefer': ['gsm_a', 'e212'], - }, - } - - # Step 1: Get all field names from tshark (F lines only, not P protocol lines) - all_field_names = [] - try: - result = subprocess.run( - ['tshark', '-G', 'fields'], - capture_output=True, text=True, timeout=15 - ) - for line in result.stdout.splitlines(): - if not line.startswith('F\t'): - continue # Only actual fields, not protocols - parts = line.split('\t') - if len(parts) >= 3: - all_field_names.append(parts[2]) - except Exception as e: - logger.warning(f"Could not query tshark fields: {e}") - - # Step 2: Search for candidate fields matching each logical name - candidates: dict[str, list[str]] = {} - for logical_name, config in search_config.items(): - matches = [] - for field_name in all_field_names: - for keyword in config['keywords']: - if keyword in field_name: - matches.append(field_name) - break - # Sort: preferred protocol prefixes first - def sort_key(name): - for i, pref in enumerate(config['prefer']): - if name.startswith(pref): - return i - return 100 - matches.sort(key=sort_key) - candidates[logical_name] = matches - - logger.info(f"tshark field candidates from -G fields: {candidates}") - - # Step 3: Validate candidates by testing with tshark -r /dev/null - # Collect all unique candidate names - all_candidate_names = set() - for field_list in candidates.values(): - all_candidate_names.update(field_list) - - valid_fields = set() - if all_candidate_names: - # Test in batches to identify which are valid - # Test each individually since batch testing makes it hard to identify valid ones - for field_name in all_candidate_names: - try: - result = subprocess.run( - ['tshark', '-T', 'fields', '-e', field_name, '-r', '/dev/null'], - capture_output=True, text=True, timeout=5 - ) - if result.returncode == 0 or 'aren\'t valid' not in result.stderr: - valid_fields.add(field_name) - except Exception: - pass - - logger.info(f"Validated tshark -e fields: {sorted(valid_fields)}") - - # Step 4: Resolve each logical field to the first valid candidate - resolved: dict[str, str | None] = {} - for logical_name, field_candidates in candidates.items(): - resolved[logical_name] = None - for candidate in field_candidates: - if candidate in valid_fields: - resolved[logical_name] = candidate - break - if resolved[logical_name] is None and field_candidates: - logger.warning( - f"No valid tshark field for '{logical_name}'. " - f"Candidates were: {field_candidates}" - ) - - logger.info(f"Resolved tshark fields: {resolved}") - _tshark_fields_cache = resolved - return resolved - - -def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subprocess.Popen, subprocess.Popen, list[str]]: - """Start grgsm_livemon and tshark processes for monitoring an ARFCN. - - Returns: - Tuple of (grgsm_process, tshark_process, field_order) - field_order is the list of logical field names in tshark column order. - - Raises: - FileNotFoundError: If grgsm_livemon or tshark not found - RuntimeError: If grgsm_livemon or tshark exits immediately - """ - frequency_hz = arfcn_to_frequency(arfcn) - frequency_mhz = frequency_hz / 1e6 - - # Check prerequisites - if not shutil.which('grgsm_livemon'): - raise FileNotFoundError('grgsm_livemon not found. Please install gr-gsm.') - - # Start grgsm_livemon - grgsm_cmd = [ - 'grgsm_livemon', - '--args', f'rtl={device_index}', - '-f', f'{frequency_mhz}M' - ] - env = dict(os.environ, - OSMO_FSM_DUP_CHECK_DISABLED='1', - PYTHONUNBUFFERED='1', - QT_QPA_PLATFORM='offscreen') - logger.info(f"Starting grgsm_livemon: {' '.join(grgsm_cmd)}") - grgsm_proc = subprocess.Popen( - grgsm_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=env - ) - register_process(grgsm_proc) - logger.info(f"Started grgsm_livemon (PID: {grgsm_proc.pid})") - - # Wait and check it didn't die immediately - time.sleep(2) - - if grgsm_proc.poll() is not None: - # Process already exited - capture stderr for diagnostics - stderr_output = '' - try: - stderr_output = grgsm_proc.stderr.read() - except Exception: - pass - exit_code = grgsm_proc.returncode - logger.error( - f"grgsm_livemon exited immediately (code: {exit_code}). " - f"stderr: {stderr_output[:500]}" - ) - unregister_process(grgsm_proc) - raise RuntimeError( - f'grgsm_livemon failed (exit code {exit_code}): {stderr_output[:200]}' - ) - - # Start stderr reader thread for grgsm_livemon diagnostics - def read_livemon_stderr(): - try: - for line in iter(grgsm_proc.stderr.readline, ''): - if line: - logger.debug(f"grgsm_livemon stderr: {line.strip()}") - except Exception: - pass - threading.Thread(target=read_livemon_stderr, daemon=True).start() - - # Start tshark - if not shutil.which('tshark'): - safe_terminate(grgsm_proc) - unregister_process(grgsm_proc) - raise FileNotFoundError('tshark not found. Please install wireshark/tshark.') - - fields = _discover_tshark_fields() - - # Build field list from only valid (non-None) fields - # Track order so parser knows which column is which - field_order = [] # list of logical names in column order - tshark_cmd = [ - 'tshark', - '-i', 'lo', - '-l', # Line-buffered output for live capture - '-f', 'udp port 4729', # Capture filter: only GSMTAP packets - ] - - # No display filter (-Y) — the capture filter (-f 'udp port 4729') - # already limits to GSMTAP packets, and the parser discards rows - # without TMSI/IMSI. A -Y filter on gsm_a.tmsi misses paging - # requests where the TMSI lives under a different field path. - tshark_cmd.extend(['-T', 'fields']) - - # Add -e for each available field in known order - for logical_name in ['ta', 'tmsi', 'imsi', 'lac', 'cid']: - if fields.get(logical_name): - tshark_cmd.extend(['-e', fields[logical_name]]) - field_order.append(logical_name) - - if not field_order: - safe_terminate(grgsm_proc) - unregister_process(grgsm_proc) - raise RuntimeError('No valid tshark fields found for GSM capture') - - logger.info(f"tshark field order: {field_order}") - logger.info(f"Starting tshark: {' '.join(tshark_cmd)}") - tshark_proc = subprocess.Popen( - tshark_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - bufsize=1 - ) - register_process(tshark_proc) - logger.info(f"Started tshark (PID: {tshark_proc.pid})") - - # Check tshark didn't exit immediately - time.sleep(1) - if tshark_proc.poll() is not None: - stderr_output = '' - try: - stderr_output = tshark_proc.stderr.read() - except Exception: - pass - exit_code = tshark_proc.returncode - logger.error(f"tshark exited immediately (code: {exit_code}). stderr: {stderr_output[:500]}") - # Clean up grgsm_livemon since monitoring can't work without tshark - safe_terminate(grgsm_proc) - unregister_process(grgsm_proc) - unregister_process(tshark_proc) - raise RuntimeError(f'tshark failed (exit code {exit_code}): {stderr_output[:200]}') - - return grgsm_proc, tshark_proc, field_order - - -def _start_and_register_monitor(arfcn: int, device_index: int) -> None: - """Start monitoring processes and register them in global state. - - This is shared logic between start_monitor() and auto_start_monitor(). - Must be called within gsm_spy_lock context. - - Args: - arfcn: ARFCN to monitor - device_index: SDR device index - """ - # Start monitoring processes - grgsm_proc, tshark_proc, field_order = _start_monitoring_processes(arfcn, device_index) - app_module.gsm_spy_livemon_process = grgsm_proc - app_module.gsm_spy_monitor_process = tshark_proc - app_module.gsm_spy_selected_arfcn = arfcn - - # Start monitoring thread - monitor_thread_obj = threading.Thread( - target=monitor_thread, - args=(tshark_proc, field_order), - daemon=True - ) - monitor_thread_obj.start() - - -@gsm_spy_bp.route('/dashboard') -def dashboard(): - """Render GSM Spy dashboard.""" - return render_template( - 'gsm_spy_dashboard.html', - shared_observer_location=SHARED_OBSERVER_LOCATION_ENABLED - ) - - -@gsm_spy_bp.route('/start', methods=['POST']) -def start_scanner(): - """Start GSM scanner (G-01 BTS Scanner).""" - global gsm_towers_found, gsm_connected - - with app_module.gsm_spy_lock: - if app_module.gsm_spy_scanner_running: - return jsonify({'error': 'Scanner already running'}), 400 - - data = request.get_json() or {} - device_index = data.get('device', 0) - region = data.get('region', 'Americas') - selected_bands = data.get('bands', []) # Get user-selected bands - - # Validate device index - try: - device_index = validate_device_index(device_index) - except ValueError as e: - return jsonify({'error': str(e)}), 400 - - # Claim SDR device to prevent conflicts - from app import claim_sdr_device - claim_error = claim_sdr_device(device_index, 'GSM Spy') - if claim_error: - return jsonify({ - 'error': claim_error, - 'error_type': 'DEVICE_BUSY' - }), 409 - - # If no bands selected, use all bands for the region (backwards compatibility) - if selected_bands: - validated_bands, error = validate_band_names(selected_bands, region) - if error: - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': error}), 400 - selected_bands = validated_bands - else: - region_bands = REGIONAL_BANDS.get(region, REGIONAL_BANDS['Americas']) - selected_bands = list(region_bands.keys()) - logger.warning(f"No bands specified, using all bands for {region}: {selected_bands}") - - # Build grgsm_scanner command - # Example: grgsm_scanner --args="rtl=0" -b GSM900 - if not shutil.which('grgsm_scanner'): - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': 'grgsm_scanner not found. Please install gr-gsm.'}), 500 - - try: - cmd = ['grgsm_scanner'] - - # Add device argument (--args for RTL-SDR device selection) - cmd.extend(['--args', f'rtl={device_index}']) - - # Add selected band arguments - # Map internal band names to grgsm_scanner -b values - # grgsm_scanner accepts: GSM900, GSM850, DCS1800, PCS1900, GSM450, GSM480, GSM-R - GRGSM_BAND_MAP = { - 'EGSM900': 'GSM900', - 'EGSM900_EXT': None, # Covered by GSM900 scan - 'GSM850': 'GSM850', - 'GSM800': None, # Not a standard GSM band for grgsm_scanner - 'DCS1800': 'DCS1800', - 'PCS1900': 'PCS1900', - } - bands_added = set() - for band_name in selected_bands: - grgsm_band = GRGSM_BAND_MAP.get(band_name, band_name) - if grgsm_band is None: - logger.info(f"Skipping band {band_name} (not supported by grgsm_scanner)") - continue - if grgsm_band not in bands_added: - cmd.extend(['-b', grgsm_band]) - bands_added.add(grgsm_band) - - if not bands_added: - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': f'No scannable bands selected. ' - f'GSM800 and EGSM900_EXT are not supported by grgsm_scanner.'}), 400 - - logger.info(f"Starting GSM scanner: {' '.join(cmd)}") - - # Set a flag to indicate scanner should run - app_module.gsm_spy_active_device = device_index - app_module.gsm_spy_region = region - app_module.gsm_spy_scanner_running = True # Use as flag initially - - # Reset counters for new session - gsm_towers_found = 0 - gsm_devices_tracked = 0 - - # Start geocoding worker (if not already running) - start_geocoding_worker() - - # Start scanning thread (will run grgsm_scanner in a loop) - scanner_thread_obj = threading.Thread( - target=scanner_thread, - args=(cmd, device_index), - daemon=True - ) - scanner_thread_obj.start() - - gsm_connected = True - - return jsonify({ - 'status': 'started', - 'device': device_index, - 'region': region - }) - - except FileNotFoundError: - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': 'grgsm_scanner not found. Please install gr-gsm.'}), 500 - except Exception as e: - from app import release_sdr_device - release_sdr_device(device_index) - logger.error(f"Error starting GSM scanner: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/monitor', methods=['POST']) -def start_monitor(): - """Start monitoring specific tower (G-02 Decoding).""" - with app_module.gsm_spy_lock: - if app_module.gsm_spy_monitor_process: - return jsonify({'error': 'Monitor already running'}), 400 - - data = request.get_json() or {} - arfcn = data.get('arfcn') - device_index = data.get('device', app_module.gsm_spy_active_device or 0) - - if not arfcn: - return jsonify({'error': 'ARFCN required'}), 400 - - # Validate ARFCN is valid integer and in known GSM band ranges - try: - arfcn = int(arfcn) - # This will raise ValueError if ARFCN is not in any known band - arfcn_to_frequency(arfcn) - except (ValueError, TypeError) as e: - return jsonify({'error': f'Invalid ARFCN: {e}'}), 400 - - # Validate device index - try: - device_index = validate_device_index(device_index) - except ValueError as e: - return jsonify({'error': str(e)}), 400 - - try: - # Start and register monitoring (shared logic) - _start_and_register_monitor(arfcn, device_index) - - return jsonify({ - 'status': 'monitoring', - 'arfcn': arfcn, - 'device': device_index - }) - - except FileNotFoundError as e: - return jsonify({'error': f'Tool not found: {e}'}), 500 - except Exception as e: - logger.error(f"Error starting monitor: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/stop', methods=['POST']) -def stop_scanner(): - """Stop GSM scanner and monitor.""" - global gsm_connected, gsm_towers_found, gsm_devices_tracked - - with app_module.gsm_spy_lock: - killed = [] - - # Stop scanner (now just a flag, thread will see it and exit) - if app_module.gsm_spy_scanner_running: - app_module.gsm_spy_scanner_running = False - killed.append('scanner') - - # Terminate livemon process - if app_module.gsm_spy_livemon_process: - unregister_process(app_module.gsm_spy_livemon_process) - if safe_terminate(app_module.gsm_spy_livemon_process, timeout=5): - killed.append('livemon') - app_module.gsm_spy_livemon_process = None - - # Terminate monitor process - if app_module.gsm_spy_monitor_process: - unregister_process(app_module.gsm_spy_monitor_process) - if safe_terminate(app_module.gsm_spy_monitor_process, timeout=5): - killed.append('monitor') - app_module.gsm_spy_monitor_process = None - - # Release SDR device from registry - if app_module.gsm_spy_active_device is not None: - from app import release_sdr_device - release_sdr_device(app_module.gsm_spy_active_device) - app_module.gsm_spy_active_device = None - app_module.gsm_spy_selected_arfcn = None - gsm_connected = False - gsm_towers_found = 0 - gsm_devices_tracked = 0 - - return jsonify({'status': 'stopped', 'killed': killed}) - - -@gsm_spy_bp.route('/stream') -def stream(): - """SSE stream for real-time GSM updates.""" - def generate(): - """Generate SSE events.""" - logger.info("SSE stream connected - client subscribed") - - # Send current state on connect (handles reconnects and late-joining clients) - existing_towers = dict(app_module.gsm_spy_towers.items()) - logger.info(f"SSE sending {len(existing_towers)} existing towers on connect") - for key, tower_data in existing_towers.items(): - yield format_sse(tower_data) - - last_keepalive = time.time() - idle_count = 0 # Track consecutive idle checks to handle transitions - - while True: - try: - # Check if scanner/monitor are still running - # Use idle counter to avoid disconnecting during scanner→monitor transition - if not app_module.gsm_spy_scanner_running and not app_module.gsm_spy_monitor_process: - idle_count += 1 - if idle_count >= 5: # 5 seconds grace period for mode transitions - logger.info("SSE stream: no active scanner or monitor, disconnecting") - yield format_sse({'type': 'disconnected'}) - break - else: - idle_count = 0 - - # Try to get data from queue - try: - data = app_module.gsm_spy_queue.get(timeout=1) - logger.info(f"SSE sending: type={data.get('type', '?')} keys={list(data.keys())}") - yield format_sse(data) - last_keepalive = time.time() - except queue.Empty: - # Send keepalive if needed - if time.time() - last_keepalive > 30: - yield format_sse({'type': 'keepalive'}) - last_keepalive = time.time() - - except GeneratorExit: - logger.info("SSE stream: client disconnected (GeneratorExit)") - break - except Exception as e: - logger.error(f"Error in GSM stream: {e}") - yield format_sse({'type': 'error', 'message': str(e)}) - break - - response = Response(generate(), mimetype='text/event-stream') - response.headers['Cache-Control'] = 'no-cache' - response.headers['X-Accel-Buffering'] = 'no' - response.headers['Connection'] = 'keep-alive' - return response - - -@gsm_spy_bp.route('/status') -def status(): - """Get current GSM Spy status.""" - api_usage = get_api_usage_today() - return jsonify({ - 'running': bool(app_module.gsm_spy_scanner_running), - 'monitoring': app_module.gsm_spy_monitor_process is not None, - 'towers_found': gsm_towers_found, - 'devices_tracked': gsm_devices_tracked, - 'device': app_module.gsm_spy_active_device, - 'region': app_module.gsm_spy_region, - 'selected_arfcn': app_module.gsm_spy_selected_arfcn, - 'api_usage_today': api_usage, - 'api_limit': config.GSM_API_DAILY_LIMIT, - 'api_remaining': config.GSM_API_DAILY_LIMIT - api_usage, - 'api_key_configured': bool(get_opencellid_api_key()) - }) - - -@gsm_spy_bp.route('/settings/api_key', methods=['GET', 'POST']) -def settings_api_key(): - """Get or set OpenCellID API key configuration.""" - from utils.database import get_setting, set_setting - - if request.method == 'GET': - env_key = config.GSM_OPENCELLID_API_KEY - db_key = get_setting('gsm.opencellid.api_key', '') - - if env_key: - source = 'env' - configured = True - elif db_key: - source = 'database' - configured = True - else: - source = 'none' - configured = False - - usage_today = get_api_usage_today() - - return jsonify({ - 'configured': configured, - 'source': source, - 'usage_today': usage_today, - 'api_limit': config.GSM_API_DAILY_LIMIT - }) - - # POST: save key to database - data = request.get_json() or {} - key = data.get('key', '').strip() - - if not key: - return jsonify({'error': 'API key cannot be empty'}), 400 - - set_setting('gsm.opencellid.api_key', key) - logger.info("OpenCellID API key saved to database") - - return jsonify({ - 'status': 'saved', - 'configured': True, - 'source': 'database' - }) - - -@gsm_spy_bp.route('/lookup_cell', methods=['POST']) -def lookup_cell(): - """Lookup cell tower via OpenCellID (G-05).""" - data = request.get_json() or {} - mcc = data.get('mcc') - mnc = data.get('mnc') - lac = data.get('lac') - cid = data.get('cid') - - if any(v is None for v in [mcc, mnc, lac, cid]): - return jsonify({'error': 'MCC, MNC, LAC, and CID required'}), 400 - - try: - # Check local cache first - with get_db() as conn: - result = conn.execute(''' - SELECT lat, lon, azimuth, range_meters, operator, radio - FROM gsm_cells - WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? - ''', (mcc, mnc, lac, cid)).fetchone() - - if result: - return jsonify({ - 'source': 'cache', - 'lat': result['lat'], - 'lon': result['lon'], - 'azimuth': result['azimuth'], - 'range': result['range_meters'], - 'operator': result['operator'], - 'radio': result['radio'] - }) - - # Check API key and usage limit - api_key = get_opencellid_api_key() - if not api_key: - return jsonify({'error': 'OpenCellID API key not configured'}), 503 - if not can_use_api(): - current_usage = get_api_usage_today() - return jsonify({ - 'error': 'OpenCellID API daily limit reached', - 'usage_today': current_usage, - 'limit': config.GSM_API_DAILY_LIMIT - }), 429 - - # Call OpenCellID API - api_url = config.GSM_OPENCELLID_API_URL - params = { - 'key': api_key, - 'mcc': mcc, - 'mnc': mnc, - 'lac': lac, - 'cellid': cid, - 'format': 'json' - } - - response = requests.get(api_url, params=params, timeout=10) - - if response.status_code == 200: - cell_data = response.json() - - # Increment API usage counter - usage_count = increment_api_usage() - logger.info(f"OpenCellID API call #{usage_count} today") - - # Cache the result - conn.execute(''' - INSERT OR REPLACE INTO gsm_cells - (mcc, mnc, lac, cid, lat, lon, azimuth, range_meters, samples, radio, operator, last_verified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) - ''', ( - mcc, mnc, lac, cid, - cell_data.get('lat'), - cell_data.get('lon'), - cell_data.get('azimuth'), - cell_data.get('range'), - cell_data.get('samples'), - cell_data.get('radio'), - cell_data.get('operator') - )) - conn.commit() - - return jsonify({ - 'source': 'api', - 'lat': cell_data.get('lat'), - 'lon': cell_data.get('lon'), - 'azimuth': cell_data.get('azimuth'), - 'range': cell_data.get('range'), - 'operator': cell_data.get('operator'), - 'radio': cell_data.get('radio') - }) - else: - return jsonify({'error': 'Cell not found in OpenCellID'}), 404 - - except Exception as e: - logger.error(f"Error looking up cell: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/detect_rogue', methods=['POST']) -def detect_rogue(): - """Analyze and flag rogue towers (G-07).""" - data = request.get_json() or {} - tower_info = data.get('tower') - - if not tower_info: - return jsonify({'error': 'Tower info required'}), 400 - - try: - is_rogue = False - reasons = [] - - # Check if tower exists in OpenCellID - mcc = tower_info.get('mcc') - mnc = tower_info.get('mnc') - lac = tower_info.get('lac') - cid = tower_info.get('cid') - - if all(v is not None for v in [mcc, mnc, lac, cid]): - with get_db() as conn: - result = conn.execute(''' - SELECT id FROM gsm_cells - WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? - ''', (mcc, mnc, lac, cid)).fetchone() - - if not result: - is_rogue = True - reasons.append('Tower not found in OpenCellID database') - - # Check signal strength anomalies - signal = tower_info.get('signal_strength', 0) - if signal > -50: # Suspiciously strong signal - is_rogue = True - reasons.append(f'Unusually strong signal: {signal} dBm') - - # If rogue, insert into database - if is_rogue: - with get_db() as conn: - conn.execute(''' - INSERT INTO gsm_rogues - (arfcn, mcc, mnc, lac, cid, signal_strength, reason, threat_level) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ''', ( - tower_info.get('arfcn'), - mcc, mnc, lac, cid, - signal, - '; '.join(reasons), - 'high' if len(reasons) > 1 else 'medium' - )) - conn.commit() - - return jsonify({ - 'is_rogue': is_rogue, - 'reasons': reasons - }) - - except Exception as e: - logger.error(f"Error detecting rogue: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/towers') -def get_towers(): - """Get all detected towers.""" - towers = [] - for key, tower_data in app_module.gsm_spy_towers.items(): - towers.append(tower_data) - return jsonify(towers) - - -@gsm_spy_bp.route('/devices') -def get_devices(): - """Get all tracked devices (IMSI/TMSI).""" - devices = [] - for key, device_data in app_module.gsm_spy_devices.items(): - devices.append(device_data) - return jsonify(devices) - - -@gsm_spy_bp.route('/rogues') -def get_rogues(): - """Get all detected rogue towers.""" - try: - with get_db() as conn: - results = conn.execute(''' - SELECT * FROM gsm_rogues - WHERE acknowledged = 0 - ORDER BY detected_at DESC - LIMIT 50 - ''').fetchall() - - rogues = [dict(row) for row in results] - return jsonify(rogues) - except Exception as e: - logger.error(f"Error fetching rogues: {e}") - return jsonify({'error': str(e)}), 500 - - -# ============================================ -# Advanced Features (G-08 through G-12) -# ============================================ - -@gsm_spy_bp.route('/velocity', methods=['GET']) -def get_velocity_data(): - """Get velocity vectoring data for tracked devices (G-08).""" - try: - device_id = request.args.get('device_id') - minutes = int(request.args.get('minutes', 60)) # Last 60 minutes by default - - with get_db() as conn: - # Get velocity log entries - query = ''' - SELECT * FROM gsm_velocity_log - WHERE timestamp >= datetime('now', '-' || ? || ' minutes') - ''' - params = [minutes] - - if device_id: - query += ' AND device_id = ?' - params.append(device_id) - - query += ' ORDER BY timestamp DESC LIMIT 100' - - results = conn.execute(query, params).fetchall() - velocity_data = [dict(row) for row in results] - - return jsonify(velocity_data) - except Exception as e: - logger.error(f"Error fetching velocity data: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/velocity/calculate', methods=['POST']) -def calculate_velocity(): - """Calculate velocity for a device based on TA transitions (G-08).""" - data = request.get_json() or {} - device_id = data.get('device_id') - - if not device_id: - return jsonify({'error': 'device_id required'}), 400 - - try: - with get_db() as conn: - # Get last two TA readings for this device - results = conn.execute(''' - SELECT ta_value, cid, timestamp - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - ORDER BY timestamp DESC - LIMIT 2 - ''', (device_id, device_id)).fetchall() - - if len(results) < 2: - return jsonify({'velocity': 0, 'message': 'Insufficient data'}) - - curr = dict(results[0]) - prev = dict(results[1]) - - # Calculate distance change (TA * 554 meters) - curr_distance = curr['ta_value'] * config.GSM_TA_METERS_PER_UNIT - prev_distance = prev['ta_value'] * config.GSM_TA_METERS_PER_UNIT - distance_change = abs(curr_distance - prev_distance) - - # Calculate time difference - curr_time = datetime.fromisoformat(curr['timestamp']) - prev_time = datetime.fromisoformat(prev['timestamp']) - time_diff_seconds = (curr_time - prev_time).total_seconds() - - # Calculate velocity (m/s) - if time_diff_seconds > 0: - velocity = distance_change / time_diff_seconds - else: - velocity = 0 - - # Store in velocity log - conn.execute(''' - INSERT INTO gsm_velocity_log - (device_id, prev_ta, curr_ta, prev_cid, curr_cid, estimated_velocity) - VALUES (?, ?, ?, ?, ?, ?) - ''', (device_id, prev['ta_value'], curr['ta_value'], - prev['cid'], curr['cid'], velocity)) - conn.commit() - - return jsonify({ - 'device_id': device_id, - 'velocity_mps': round(velocity, 2), - 'velocity_kmh': round(velocity * 3.6, 2), - 'distance_change_m': round(distance_change, 2), - 'time_diff_s': round(time_diff_seconds, 2) - }) - - except Exception as e: - logger.error(f"Error calculating velocity: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/crowd_density', methods=['GET']) -def get_crowd_density(): - """Get crowd density data by sector (G-09).""" - try: - hours = int(request.args.get('hours', 1)) # Last 1 hour by default - cid = request.args.get('cid') # Optional: specific cell - - with get_db() as conn: - # Count unique TMSI per cell in time window - query = ''' - SELECT - cid, - lac, - COUNT(DISTINCT tmsi) as unique_devices, - COUNT(*) as total_pings, - MIN(timestamp) as first_seen, - MAX(timestamp) as last_seen - FROM gsm_tmsi_log - WHERE timestamp >= datetime('now', '-' || ? || ' hours') - ''' - params = [hours] - - if cid: - query += ' AND cid = ?' - params.append(cid) - - query += ' GROUP BY cid, lac ORDER BY unique_devices DESC' - - results = conn.execute(query, params).fetchall() - density_data = [] - - for row in results: - density_data.append({ - 'cid': row['cid'], - 'lac': row['lac'], - 'unique_devices': row['unique_devices'], - 'total_pings': row['total_pings'], - 'first_seen': row['first_seen'], - 'last_seen': row['last_seen'], - 'density_level': 'high' if row['unique_devices'] > 20 else - 'medium' if row['unique_devices'] > 10 else 'low' - }) - - return jsonify(density_data) - - except Exception as e: - logger.error(f"Error fetching crowd density: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/life_patterns', methods=['GET']) -def get_life_patterns(): - """Get life pattern analysis for a device (G-10).""" - try: - device_id = request.args.get('device_id') - if not device_id: - # Return empty results gracefully when no device selected - return jsonify({ - 'device_id': None, - 'patterns': [], - 'message': 'No device selected' - }), 200 - - with get_db() as conn: - # Get historical signal data - results = conn.execute(''' - SELECT - strftime('%H', timestamp) as hour, - strftime('%w', timestamp) as day_of_week, - cid, - lac, - COUNT(*) as occurrences - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - AND timestamp >= datetime('now', '-60 days') - GROUP BY hour, day_of_week, cid, lac - ORDER BY occurrences DESC - ''', (device_id, device_id)).fetchall() - - patterns = [] - for row in results: - patterns.append({ - 'hour': int(row['hour']), - 'day_of_week': int(row['day_of_week']), - 'cid': row['cid'], - 'lac': row['lac'], - 'occurrences': row['occurrences'], - 'day_name': ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'][int(row['day_of_week'])] - }) - - # Identify regular patterns - regular_locations = [] - for pattern in patterns[:5]: # Top 5 most frequent - if pattern['occurrences'] >= 3: # Seen at least 3 times - regular_locations.append({ - 'cid': pattern['cid'], - 'typical_time': f"{pattern['day_name']} {pattern['hour']:02d}:00", - 'frequency': pattern['occurrences'] - }) - - return jsonify({ - 'device_id': device_id, - 'patterns': patterns, - 'regular_locations': regular_locations, - 'total_observations': sum(p['occurrences'] for p in patterns) - }) - - except Exception as e: - logger.error(f"Error analyzing life patterns: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/neighbor_audit', methods=['GET']) -def neighbor_audit(): - """Audit neighbor cell lists for consistency (G-11).""" - try: - cid = request.args.get('cid') - if not cid: - # Return empty results gracefully when no tower selected - return jsonify({ - 'cid': None, - 'neighbors': [], - 'inconsistencies': [], - 'message': 'No tower selected' - }), 200 - - with get_db() as conn: - # Get tower info with metadata (neighbor list stored in metadata JSON) - result = conn.execute(''' - SELECT metadata FROM gsm_cells WHERE cid = ? - ''', (cid,)).fetchone() - - if not result or not result['metadata']: - return jsonify({ - 'cid': cid, - 'status': 'no_data', - 'message': 'No neighbor list data available' - }) - - # Parse metadata JSON - metadata = json.loads(result['metadata']) - neighbor_list = metadata.get('neighbors', []) - - # Audit consistency - issues = [] - for neighbor_cid in neighbor_list: - # Check if neighbor exists in database - neighbor_exists = conn.execute(''' - SELECT id FROM gsm_cells WHERE cid = ? - ''', (neighbor_cid,)).fetchone() - - if not neighbor_exists: - issues.append({ - 'type': 'missing_neighbor', - 'cid': neighbor_cid, - 'message': f'Neighbor CID {neighbor_cid} not found in database' - }) - - return jsonify({ - 'cid': cid, - 'neighbor_count': len(neighbor_list), - 'neighbors': neighbor_list, - 'issues': issues, - 'status': 'suspicious' if issues else 'normal' - }) - - except Exception as e: - logger.error(f"Error auditing neighbors: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/traffic_correlation', methods=['GET']) -def traffic_correlation(): - """Correlate uplink/downlink traffic for pairing analysis (G-12).""" - try: - cid = request.args.get('cid') - minutes = int(request.args.get('minutes', 5)) - - with get_db() as conn: - # Get recent signal activity for this cell - results = conn.execute(''' - SELECT - imsi, - tmsi, - ta_value, - timestamp, - metadata - FROM gsm_signals - WHERE cid = ? - AND timestamp >= datetime('now', '-' || ? || ' minutes') - ORDER BY timestamp DESC - ''', (cid, minutes)).fetchall() - - correlations = [] - seen_devices = set() - - for row in results: - device_id = row['imsi'] or row['tmsi'] - if device_id and device_id not in seen_devices: - seen_devices.add(device_id) - - # Simple correlation: count bursts - burst_count = conn.execute(''' - SELECT COUNT(*) as bursts - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - AND cid = ? - AND timestamp >= datetime('now', '-' || ? || ' minutes') - ''', (device_id, device_id, cid, minutes)).fetchone() - - correlations.append({ - 'device_id': device_id, - 'burst_count': burst_count['bursts'], - 'last_seen': row['timestamp'], - 'ta_value': row['ta_value'], - 'activity_level': 'high' if burst_count['bursts'] > 10 else - 'medium' if burst_count['bursts'] > 5 else 'low' - }) - - return jsonify({ - 'cid': cid, - 'time_window_minutes': minutes, - 'active_devices': len(correlations), - 'correlations': correlations - }) - - except Exception as e: - logger.error(f"Error correlating traffic: {e}") - return jsonify({'error': str(e)}), 500 - - -# ============================================ -# Helper Functions -# ============================================ - -def parse_grgsm_scanner_output(line: str) -> dict[str, Any] | None: - """Parse grgsm_scanner output line. - - Actual output format (comma-separated key-value pairs): - ARFCN: 975, Freq: 925.2M, CID: 13522, LAC: 38722, MCC: 262, MNC: 1, Pwr: -58 - """ - try: - line = line.strip() - - # Skip non-data lines (progress, config, neighbour info, blank) - if not line or 'ARFCN:' not in line: - return None - - # Parse "ARFCN: 975, Freq: 925.2M, CID: 13522, LAC: 38722, MCC: 262, MNC: 1, Pwr: -58" - fields = {} - for part in line.split(','): - part = part.strip() - if ':' in part: - key, _, value = part.partition(':') - fields[key.strip()] = value.strip() - - if 'ARFCN' in fields and 'CID' in fields: - cid = int(fields.get('CID', 0)) - mcc = int(fields.get('MCC', 0)) - mnc = int(fields.get('MNC', 0)) - - # Only skip entries with no network identity at all (MCC=0 AND MNC=0) - # CID=0 with valid MCC/MNC is a partially decoded cell - still useful - if mcc == 0 and mnc == 0: - logger.debug(f"Skipping unidentified ARFCN (MCC=0, MNC=0): {line}") - return None - - # Freq may have 'M' suffix (e.g. "925.2M") - freq_str = fields.get('Freq', '0').rstrip('Mm') - - data = { - 'type': 'tower', - 'arfcn': int(fields['ARFCN']), - 'frequency': float(freq_str), - 'cid': cid, - 'lac': int(fields.get('LAC', 0)), - 'mcc': mcc, - 'mnc': mnc, - 'signal_strength': float(fields.get('Pwr', -999)), - 'timestamp': datetime.now().isoformat() - } - return data - - except Exception as e: - logger.debug(f"Failed to parse scanner line: {line} - {e}") - - return None - - -def parse_tshark_output(line: str, field_order: list[str] | None = None) -> dict[str, Any] | None: - """Parse tshark filtered GSM output. - - Args: - line: Tab-separated tshark output line - field_order: List of logical field names in column order. - If None, assumes legacy order: ['ta', 'tmsi', 'imsi', 'lac', 'cid'] - """ - if field_order is None: - field_order = ['ta', 'tmsi', 'imsi', 'lac', 'cid'] - - try: - parts = line.rstrip('\n\r').split('\t') - - if len(parts) < len(field_order): - return None - - # Map logical names to column values - field_map = {} - for i, logical_name in enumerate(field_order): - field_map[logical_name] = parts[i] if parts[i] else None - - # Convert types (use base 0 to auto-detect hex 0x prefix from tshark) - ta_raw = field_map.get('ta') - data = { - 'type': 'device', - 'ta_value': int(ta_raw, 0) if ta_raw else None, - 'tmsi': field_map.get('tmsi'), - 'imsi': field_map.get('imsi'), - 'lac': int(field_map['lac'], 0) if field_map.get('lac') else None, - 'cid': int(field_map['cid'], 0) if field_map.get('cid') else None, - 'timestamp': datetime.now().isoformat() - } - - # Need at least one identifier - if not data['tmsi'] and not data['imsi']: - return None - - # Calculate distance from TA - if data['ta_value'] is not None: - data['distance_meters'] = data['ta_value'] * config.GSM_TA_METERS_PER_UNIT - - return data - - except Exception as e: - logger.debug(f"Failed to parse tshark line: {line} - {e}") - - return None - - -def auto_start_monitor(tower_data): - """Automatically start monitoring the strongest tower found.""" - try: - arfcn = tower_data.get('arfcn') - if not arfcn: - logger.warning("Cannot auto-monitor: no ARFCN in tower data") - return - - logger.info(f"Auto-monitoring strongest tower: ARFCN {arfcn}, Signal {tower_data.get('signal_strength')} dBm") - - # Brief delay to ensure scanner has stabilized - time.sleep(2) - - with app_module.gsm_spy_lock: - if app_module.gsm_spy_monitor_process: - logger.info("Monitor already running, skipping auto-start") - return - - device_index = app_module.gsm_spy_active_device or 0 - - # Start and register monitoring (shared logic) - _start_and_register_monitor(arfcn, device_index) - - # Send SSE notification - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'auto_monitor_started', - 'arfcn': arfcn, - 'tower': tower_data - }) - except queue.Full: - pass - - logger.info(f"Auto-monitoring started for ARFCN {arfcn}") - - except Exception as e: - logger.error(f"Error in auto-monitoring: {e}") - - -def scanner_thread(cmd, device_index): - """Thread to continuously run grgsm_scanner in a loop with non-blocking I/O. - - grgsm_scanner scans once and exits, so we loop it to provide - continuous updates to the dashboard. - """ - global gsm_towers_found - - strongest_tower = None - auto_monitor_triggered = False # Moved outside loop - persists across scans - scan_count = 0 - crash_count = 0 - process = None - - try: - while app_module.gsm_spy_scanner_running: # Flag check - scan_count += 1 - logger.info(f"Starting GSM scan #{scan_count}") - - try: - # Start scanner process - # Set OSMO_FSM_DUP_CHECK_DISABLED to prevent libosmocore - # abort on duplicate FSM registration (common with apt gr-gsm) - env = dict(os.environ, - OSMO_FSM_DUP_CHECK_DISABLED='1', - PYTHONUNBUFFERED='1', - QT_QPA_PLATFORM='offscreen') - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - bufsize=1, - env=env - ) - register_process(process) - logger.info(f"Started grgsm_scanner (PID: {process.pid})") - - # Standard pattern: reader threads with queue - output_queue_local = queue.Queue() - - def read_stdout(): - try: - for line in iter(process.stdout.readline, ''): - if line: - output_queue_local.put(('stdout', line)) - except Exception as e: - logger.error(f"stdout read error: {e}") - finally: - output_queue_local.put(('eof', None)) - - def read_stderr(): - try: - for line in iter(process.stderr.readline, ''): - if line: - logger.debug(f"grgsm_scanner stderr: {line.strip()}") - # grgsm_scanner outputs scan results to stderr - output_queue_local.put(('stderr', line)) - except Exception as e: - logger.error(f"stderr read error: {e}") - - stdout_thread = threading.Thread(target=read_stdout, daemon=True) - stderr_thread = threading.Thread(target=read_stderr, daemon=True) - stdout_thread.start() - stderr_thread.start() - - # Process output with timeout - scan_start = time.time() - last_output = scan_start - scan_timeout = 300 # 5 minute maximum per scan (4 bands takes ~2-3 min) - - while app_module.gsm_spy_scanner_running: - # Check if process died - if process.poll() is not None: - logger.info(f"Scanner exited (code: {process.returncode})") - break - - # Get output from queue with timeout - try: - msg_type, line = output_queue_local.get(timeout=1.0) - - if msg_type == 'eof': - break # EOF - - last_output = time.time() - stripped = line.strip() - logger.info(f"Scanner [{msg_type}]: {stripped}") - - # Forward progress and status info to frontend - progress_match = re.match(r'Scanning:\s+([\d.]+)%\s+done', stripped) - if progress_match: - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'progress', - 'percent': float(progress_match.group(1)), - 'scan': scan_count - }) - except queue.Full: - pass - continue - if stripped.startswith('Try scan CCCH'): - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'status', - 'message': stripped, - 'scan': scan_count - }) - except queue.Full: - pass - - parsed = parse_grgsm_scanner_output(line) - if parsed: - # Enrich with coordinates - from utils.gsm_geocoding import enrich_tower_data - enriched = enrich_tower_data(parsed) - - # Store in DataStore - key = f"{enriched['mcc']}_{enriched['mnc']}_{enriched['lac']}_{enriched['cid']}" - app_module.gsm_spy_towers[key] = enriched - - # Track strongest tower - signal = enriched.get('signal_strength', -999) - if strongest_tower is None or signal > strongest_tower.get('signal_strength', -999): - strongest_tower = enriched - - # Queue for SSE - try: - app_module.gsm_spy_queue.put_nowait(enriched) - except queue.Full: - logger.warning("Queue full, dropping tower update") - - # Thread-safe counter update - with app_module.gsm_spy_lock: - gsm_towers_found += 1 - except queue.Empty: - # No output, check timeout - if time.time() - last_output > scan_timeout: - logger.warning(f"Scan timeout after {scan_timeout}s") - break - - # Drain remaining queue items after process exits - while not output_queue_local.empty(): - try: - msg_type, line = output_queue_local.get_nowait() - if line: - logger.info(f"Scanner [{msg_type}] (drain): {line.strip()}") - except queue.Empty: - break - - # Clean up process with timeout - if process.poll() is None: - logger.info("Terminating scanner process") - safe_terminate(process, timeout=5) - else: - process.wait() # Reap zombie - - exit_code = process.returncode - scan_duration = time.time() - scan_start - logger.info(f"Scan #{scan_count} complete (exit code: {exit_code}, duration: {scan_duration:.1f}s)") - - # Notify frontend scan completed - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'scan_complete', - 'scan': scan_count, - 'duration': round(scan_duration, 1), - 'towers_found': gsm_towers_found - }) - except queue.Full: - pass - - # Detect crash pattern: process exits too quickly with no data - if scan_duration < 5 and exit_code != 0: - crash_count += 1 - logger.error( - f"grgsm_scanner crashed on startup (exit code: {exit_code}). " - f"Crash count: {crash_count}. Check gr-gsm/libosmocore compatibility." - ) - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'error', - 'message': f'grgsm_scanner crashed (exit code: {exit_code}). ' - 'This may be a gr-gsm/libosmocore compatibility issue. ' - 'Try rebuilding gr-gsm from source.', - 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S') - }) - except queue.Full: - pass - if crash_count >= 3: - logger.error("grgsm_scanner crashed 3 times, stopping scanner") - break - - except FileNotFoundError: - logger.error( - "grgsm_scanner not found. Please install gr-gsm: " - "https://github.com/bkerler/gr-gsm" - ) - # Send error to SSE stream so the UI knows - try: - app_module.gsm_spy_queue.put({ - 'type': 'error', - 'message': 'grgsm_scanner not found. Please install gr-gsm.', - 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S') - }) - except Exception: - pass - break # Don't retry - binary won't appear - - except Exception as e: - logger.error(f"Scanner scan error: {e}", exc_info=True) - if process and process.poll() is None: - safe_terminate(process) - - # Check if should continue - if not app_module.gsm_spy_scanner_running: - break - - # After first scan completes: auto-switch to monitoring if towers found - # Scanner process has exited so SDR is free for grgsm_livemon - if not auto_monitor_triggered and strongest_tower and scan_count >= 1: - auto_monitor_triggered = True - arfcn = strongest_tower.get('arfcn') - signal = strongest_tower.get('signal_strength', -999) - logger.info( - f"Scan complete with towers found. Auto-switching to monitor mode " - f"on ARFCN {arfcn} (signal: {signal} dBm)" - ) - - # Stop scanner loop - SDR needed for monitoring - app_module.gsm_spy_scanner_running = False - - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'status', - 'message': f'Switching to monitor mode on ARFCN {arfcn}...' - }) - except queue.Full: - pass - - # Start monitoring (SDR is free since scanner process exited) - try: - with app_module.gsm_spy_lock: - if not app_module.gsm_spy_monitor_process: - _start_and_register_monitor(arfcn, device_index) - logger.info(f"Auto-monitoring started for ARFCN {arfcn}") - - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'auto_monitor_started', - 'arfcn': arfcn, - 'tower': strongest_tower - }) - except queue.Full: - pass - except Exception as e: - logger.error(f"Error starting auto-monitor: {e}", exc_info=True) - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'error', - 'message': f'Monitor failed: {e}' - }) - except queue.Full: - pass - # Resume scanning if monitor failed - app_module.gsm_spy_scanner_running = True - - break # Exit scanner loop (monitoring takes over) - - # Wait between scans with responsive flag checking - logger.info("Waiting 5 seconds before next scan") - for i in range(5): - if not app_module.gsm_spy_scanner_running: - break - time.sleep(1) - - except Exception as e: - logger.error(f"Scanner thread fatal error: {e}", exc_info=True) - - finally: - # Always cleanup - if process and process.poll() is None: - safe_terminate(process, timeout=5) - - logger.info("Scanner thread terminated") - - # Reset global state - but don't release SDR if monitoring took over - with app_module.gsm_spy_lock: - app_module.gsm_spy_scanner_running = False - if app_module.gsm_spy_monitor_process is None: - # No monitor running - release SDR device - if app_module.gsm_spy_active_device is not None: - from app import release_sdr_device - release_sdr_device(app_module.gsm_spy_active_device) - app_module.gsm_spy_active_device = None - else: - logger.info("Monitor is running, keeping SDR device allocated") - - -def monitor_thread(process, field_order=None): - """Thread to read tshark output using standard iter pattern.""" - global gsm_devices_tracked - - # Standard pattern: reader thread with queue - output_queue_local = queue.Queue() - - def read_stdout(): - try: - for line in iter(process.stdout.readline, ''): - if line: - output_queue_local.put(('stdout', line)) - except Exception as e: - logger.error(f"tshark read error: {e}") - finally: - output_queue_local.put(('eof', None)) - - def read_stderr(): - try: - for line in iter(process.stderr.readline, ''): - if line: - logger.debug(f"tshark stderr: {line.strip()}") - except Exception: - pass - - stdout_thread = threading.Thread(target=read_stdout, daemon=True) - stdout_thread.start() - stderr_thread = threading.Thread(target=read_stderr, daemon=True) - stderr_thread.start() - - monitor_start_time = time.time() - packets_captured = 0 - lines_received = 0 - last_heartbeat = time.time() - - try: - while app_module.gsm_spy_monitor_process: - # Check if process died - if process.poll() is not None: - logger.info(f"Monitor process exited (code: {process.returncode})") - break - - # Send periodic heartbeat so frontend knows monitor is alive - now = time.time() - if now - last_heartbeat >= 5: - last_heartbeat = now - elapsed = int(now - monitor_start_time) - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'monitor_heartbeat', - 'elapsed': elapsed, - 'packets': packets_captured, - 'devices': len(app_module.gsm_spy_devices) - }) - except queue.Full: - pass - # Periodic diagnostic: how many raw lines vs parsed - if lines_received > 0 or elapsed % 30 == 0: - logger.info( - f"Monitor stats: {lines_received} tshark lines received, " - f"{packets_captured} parsed, fields={field_order}" - ) - - # Get output from queue with timeout - try: - msg_type, line = output_queue_local.get(timeout=1.0) - except queue.Empty: - continue # Timeout, check flag again - - if msg_type == 'eof': - break # EOF - - lines_received += 1 - # Log first 5 raw lines and then every 100th for diagnostics - if lines_received <= 5 or lines_received % 100 == 0: - logger.debug(f"tshark raw line #{lines_received}: {line.rstrip()!r}") - - parsed = parse_tshark_output(line, field_order) - if parsed: - packets_captured += 1 - - # Store in DataStore - key = parsed.get('tmsi') or parsed.get('imsi') or str(time.time()) - app_module.gsm_spy_devices[key] = parsed - - # Queue for SSE stream - try: - app_module.gsm_spy_queue.put_nowait(parsed) - except queue.Full: - pass - - # Store in database for historical analysis - try: - with get_db() as conn: - # gsm_signals table - conn.execute(''' - INSERT INTO gsm_signals - (imsi, tmsi, lac, cid, ta_value, arfcn) - VALUES (?, ?, ?, ?, ?, ?) - ''', ( - parsed.get('imsi'), - parsed.get('tmsi'), - parsed.get('lac'), - parsed.get('cid'), - parsed.get('ta_value'), - app_module.gsm_spy_selected_arfcn - )) - - # gsm_tmsi_log table for crowd density - if parsed.get('tmsi'): - conn.execute(''' - INSERT INTO gsm_tmsi_log - (tmsi, lac, cid, ta_value) - VALUES (?, ?, ?, ?) - ''', ( - parsed.get('tmsi'), - parsed.get('lac'), - parsed.get('cid'), - parsed.get('ta_value') - )) - - # Velocity calculation (G-08) - device_id = parsed.get('imsi') or parsed.get('tmsi') - if device_id and parsed.get('ta_value') is not None: - # Get previous TA reading - prev_reading = conn.execute(''' - SELECT ta_value, cid, timestamp - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - ORDER BY timestamp DESC - LIMIT 1 OFFSET 1 - ''', (device_id, device_id)).fetchone() - - if prev_reading: - # Calculate velocity - curr_ta = parsed.get('ta_value') - prev_ta = prev_reading['ta_value'] - curr_distance = curr_ta * config.GSM_TA_METERS_PER_UNIT - prev_distance = prev_ta * config.GSM_TA_METERS_PER_UNIT - distance_change = abs(curr_distance - prev_distance) - - # Time difference - prev_time = datetime.fromisoformat(prev_reading['timestamp']) - curr_time = datetime.now() - time_diff_seconds = (curr_time - prev_time).total_seconds() - - if time_diff_seconds > 0: - velocity = distance_change / time_diff_seconds - - # Store velocity - conn.execute(''' - INSERT INTO gsm_velocity_log - (device_id, prev_ta, curr_ta, prev_cid, curr_cid, estimated_velocity) - VALUES (?, ?, ?, ?, ?, ?) - ''', ( - device_id, - prev_ta, - curr_ta, - prev_reading['cid'], - parsed.get('cid'), - velocity - )) - - conn.commit() - except Exception as e: - logger.error(f"Error storing device data: {e}") - - # Thread-safe counter - with app_module.gsm_spy_lock: - gsm_devices_tracked += 1 - - except Exception as e: - logger.error(f"Monitor thread error: {e}", exc_info=True) - - finally: - # Reap process with timeout - try: - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - logger.warning("Monitor process didn't terminate, killing") - process.kill() - process.wait() - else: - process.wait() - logger.info(f"Monitor process exited with code {process.returncode}") - except Exception as e: - logger.error(f"Error reaping monitor process: {e}") - - logger.info("Monitor thread terminated") diff --git a/setup.sh b/setup.sh index 3e0f210..f91e2b0 100755 --- a/setup.sh +++ b/setup.sh @@ -243,12 +243,6 @@ check_tools() { check_required "hcitool" "Bluetooth scan utility" hcitool check_required "hciconfig" "Bluetooth adapter config" hciconfig - echo - info "GSM Intelligence:" - check_recommended "grgsm_scanner" "GSM tower scanner (gr-gsm)" grgsm_scanner - check_recommended "grgsm_livemon" "GSM live monitor (gr-gsm)" grgsm_livemon - check_recommended "tshark" "Packet analysis (Wireshark)" tshark - echo info "SoapySDR:" check_required "SoapySDRUtil" "SoapySDR CLI utility" SoapySDRUtil @@ -713,47 +707,6 @@ install_macos_packages() { progress "Installing gpsd" brew_install gpsd - # gr-gsm for GSM Intelligence - progress "Installing gr-gsm" - if ! cmd_exists grgsm_scanner; then - brew_install gnuradio - (brew_install gr-gsm) || { - warn "gr-gsm not available in Homebrew, building from source..." - ( - tmp_dir="$(mktemp -d)" - trap 'rm -rf "$tmp_dir"' EXIT - - info "Cloning gr-gsm repository..." - git clone --depth 1 https://github.com/bkerler/gr-gsm.git "$tmp_dir/gr-gsm" >/dev/null 2>&1 \ - || { warn "Failed to clone gr-gsm. GSM Spy feature will not work."; exit 1; } - - cd "$tmp_dir/gr-gsm" - mkdir -p build && cd build - info "Compiling gr-gsm (this may take several minutes)..." - if cmake .. >/dev/null 2>&1 && make -j$(sysctl -n hw.ncpu) >/dev/null 2>&1; then - if [[ -w /usr/local/lib ]]; then - make install >/dev/null 2>&1 - else - sudo make install >/dev/null 2>&1 - fi - ok "gr-gsm installed successfully from source" - else - warn "Failed to build gr-gsm. GSM Spy feature will not work." - fi - ) - } - else - ok "gr-gsm already installed" - fi - - # Wireshark (tshark) for GSM packet analysis - progress "Installing tshark" - if ! cmd_exists tshark; then - brew_install wireshark - else - ok "tshark already installed" - fi - progress "Installing Ubertooth tools (optional)" if ! cmd_exists ubertooth-btle; then echo @@ -1164,82 +1117,6 @@ install_debian_packages() { progress "Installing gpsd" apt_install gpsd gpsd-clients || true - # gr-gsm for GSM Intelligence - progress "Installing GNU Radio and gr-gsm" - if ! cmd_exists grgsm_scanner; then - # Try to install gr-gsm directly from package repositories - apt_install gnuradio gnuradio-dev gr-osmosdr gr-gsm || { - warn "gr-gsm package not available in repositories. Attempting source build..." - - # Fallback: Build from source - progress "Building gr-gsm from source" - apt_install git cmake libboost-all-dev libcppunit-dev swig \ - doxygen liblog4cpp5-dev python3-scipy python3-numpy \ - libvolk-dev libuhd-dev libfftw3-dev || true - - info "Cloning gr-gsm repository..." - if [ -d /tmp/gr-gsm ]; then - rm -rf /tmp/gr-gsm - fi - - git clone https://github.com/bkerler/gr-gsm.git /tmp/gr-gsm || { - warn "Failed to clone gr-gsm repository. GSM Spy will not be available." - return 0 - } - - cd /tmp/gr-gsm - mkdir -p build && cd build - - # Try to find GNU Radio cmake files - if [ -d /usr/lib/x86_64-linux-gnu/cmake/gnuradio ]; then - export CMAKE_PREFIX_PATH="/usr/lib/x86_64-linux-gnu/cmake/gnuradio:$CMAKE_PREFIX_PATH" - fi - - info "Running CMake configuration..." - if cmake .. 2>/dev/null; then - info "Compiling gr-gsm (this may take several minutes)..." - if make -j$(nproc) 2>/dev/null; then - $SUDO make install - $SUDO ldconfig - cd ~ - rm -rf /tmp/gr-gsm - ok "gr-gsm built and installed successfully" - else - warn "gr-gsm compilation failed. GSM Spy feature will not work." - cd ~ - rm -rf /tmp/gr-gsm - fi - else - warn "gr-gsm CMake configuration failed. GNU Radio 3.8+ may not be available." - cd ~ - rm -rf /tmp/gr-gsm - fi - } - - # Verify installation - if cmd_exists grgsm_scanner; then - ok "gr-gsm installed successfully" - else - warn "gr-gsm installation incomplete. GSM Spy feature will not work." - fi - else - ok "gr-gsm already installed" - fi - - # Wireshark (tshark) for GSM packet analysis - progress "Installing tshark" - if ! cmd_exists tshark; then - # Pre-accept non-root capture prompt for non-interactive install - echo 'wireshark-common wireshark-common/install-setuid boolean true' | $SUDO debconf-set-selections - apt_install tshark || true - # Allow non-root capture - $SUDO dpkg-reconfigure wireshark-common 2>/dev/null || true - $SUDO usermod -a -G wireshark $USER 2>/dev/null || true - ok "tshark installed. You may need to re-login for wireshark group permissions." - else - ok "tshark already installed" - fi - progress "Installing Python packages" apt_install python3-venv python3-pip || true # Install Python packages via apt (more reliable than pip on modern Debian/Ubuntu) @@ -1327,7 +1204,7 @@ final_summary_and_hard_fail() { warn "Missing RECOMMENDED tools (some features will not work):" for t in "${missing_recommended[@]}"; do echo " - $t"; done echo - warn "Install these for full functionality (GSM Intelligence, etc.)" + warn "Install these for full functionality" fi } @@ -1376,7 +1253,7 @@ main() { install_python_deps - # Download leaflet-heat plugin for GSM heatmap (offline mode) + # Download leaflet-heat plugin (offline mode) if [ ! -f "static/vendor/leaflet-heat/leaflet-heat.js" ]; then info "Downloading leaflet-heat plugin..." mkdir -p static/vendor/leaflet-heat diff --git a/static/js/core/settings-manager.js b/static/js/core/settings-manager.js index 69ed89b..35649c2 100644 --- a/static/js/core/settings-manager.js +++ b/static/js/core/settings-manager.js @@ -946,32 +946,9 @@ function loadApiKeyStatus() { if (!badge) return; - fetch('/gsm_spy/settings/api_key') - .then(r => r.json()) - .then(data => { - if (data.configured) { - badge.textContent = 'Configured'; - badge.className = 'asset-badge available'; - desc.textContent = 'Source: ' + (data.source === 'env' ? 'Environment variable' : 'Database'); - } else { - badge.textContent = 'Not configured'; - badge.className = 'asset-badge missing'; - desc.textContent = 'No API key set'; - } - if (usage) { - usage.textContent = (data.usage_today || 0) + ' / ' + (data.api_limit || 1000); - } - if (bar) { - const pct = Math.min(100, ((data.usage_today || 0) / (data.api_limit || 1000)) * 100); - bar.style.width = pct + '%'; - bar.style.background = pct > 90 ? 'var(--accent-red)' : pct > 70 ? 'var(--accent-yellow)' : 'var(--accent-cyan)'; - } - }) - .catch(() => { - badge.textContent = 'Error'; - badge.className = 'asset-badge missing'; - desc.textContent = 'Could not load status'; - }); + badge.textContent = 'Not available'; + badge.className = 'asset-badge missing'; + desc.textContent = 'GSM feature removed'; } /** @@ -994,30 +971,8 @@ function saveApiKey() { result.style.color = 'var(--text-dim)'; result.textContent = 'Saving...'; - fetch('/gsm_spy/settings/api_key', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ key: key }) - }) - .then(r => r.json()) - .then(data => { - if (data.error) { - result.style.color = 'var(--accent-red)'; - result.textContent = data.error; - } else { - result.style.color = 'var(--accent-green)'; - result.textContent = 'API key saved successfully.'; - input.value = ''; - loadApiKeyStatus(); - // Hide the banner if visible - const banner = document.getElementById('apiKeyBanner'); - if (banner) banner.style.display = 'none'; - } - }) - .catch(() => { - result.style.color = 'var(--accent-red)'; - result.textContent = 'Error saving API key.'; - }); + result.style.color = 'var(--accent-red)'; + result.textContent = 'GSM feature has been removed.'; } /** diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html deleted file mode 100644 index 85b12a2..0000000 --- a/templates/gsm_spy_dashboard.html +++ /dev/null @@ -1,3300 +0,0 @@ - - - - - - GSM SPY // INTERCEPT - See the Invisible - - {% if offline_settings.fonts_source == 'local' %} - - {% else %} - - {% endif %} - - {% if offline_settings.assets_source == 'local' %} - - - {% else %} - - - {% endif %} - - {% if offline_settings.assets_source == 'local' %} - - {% else %} - - {% endif %} - - - - - - - - - - - -
-
- -
- -
-
- STANDBY -
-
- - {% set active_mode = 'gsm' %} - {% include 'partials/nav.html' with context %} - - - - - {% include 'partials/settings-modal.html' %} - - -
-
-
- 0 - TOWERS -
-
- 0 - DEVICES -
-
- 0 - ROGUES -
-
- 0 - SIGNALS -
-
- - - CROWD -
-
-
-
- STANDBY -
-
--:--:-- UTC
- - -
-
- - -
-
-
-
Analytics Overview
- -
-
-
- -
-
-
📍
-
Velocity Tracking
-
-
- Track device movement by analyzing Timing Advance transitions and cell handovers. - Estimates velocity and direction based on TA delta and cell sector patterns. -
-
-
-
0
-
Devices Tracked
-
-
-
- km/h
-
Avg Velocity
-
-
-
- - -
-
-
👥
-
Crowd Density
-
-
- Aggregate TMSI pings per cell sector to estimate crowd density. - Visualizes hotspots and congestion patterns across towers. -
-
-
-
0
-
Total Devices
-
-
-
0
-
Peak Sector
-
-
-
- - -
-
-
📊
-
Life Patterns
-
-
- Analyze 60-day historical data to identify recurring patterns in device behavior. - Detects work locations, commute routes, and daily routines. -
-
-
-
0
-
Patterns Found
-
-
-
0%
-
Confidence
-
-
-
- - -
-
-
🔍
-
Neighbor Audit
-
-
- Validate neighbor cell lists against expected network topology. - Detects inconsistencies that may indicate rogue towers. -
-
-
-
0
-
Neighbors
-
-
-
0
-
Anomalies
-
-
-
- - -
-
-
📡
-
Traffic Correlation
-
-
- Correlate uplink and downlink timing to identify communication patterns. - Maps device-to-device interactions and network flows. -
-
-
-
0
-
Paired Flows
-
-
-
0
-
Active Now
-
-
-
-
-
-
-
- -
- - - - -
- - -
-
- - - - - -
- -
- GPS LOCATION -
- - - -
-
- - -
- GSM SCANNER -
- - - -
-
-
-
- - -
-
-
-
- Device Detail - -
-
-
-
-
- - - - - - - - - diff --git a/templates/index.html b/templates/index.html index 373005e..7d635ef 100644 --- a/templates/index.html +++ b/templates/index.html @@ -171,10 +171,6 @@ Vessels - - - GSM SPY - - @@ -360,70 +359,6 @@ - -
-
-
OpenCellID API Key
-

- Required for GSM cell tower geolocation. Get a free key at - opencellid.org/register - (1,000 lookups/day). -

- -
-
- Status - Checking... -
- Checking... -
- -
-
- API Key - Paste your OpenCellID API token -
-
- - -
-
- -
- -
- - -
- -
-
Usage Today
-
-
- API Calls - -- / -- -
-
-
-
-
-
- -
- Note: The environment variable INTERCEPT_GSM_OPENCELLID_API_KEY takes priority over the saved key. - Keys saved here persist across restarts. -
-
-
diff --git a/tests/test_gsm_spy.py b/tests/test_gsm_spy.py deleted file mode 100644 index 797f794..0000000 --- a/tests/test_gsm_spy.py +++ /dev/null @@ -1,360 +0,0 @@ -"""Unit tests for GSM Spy parsing and validation functions.""" - -import pytest -from routes.gsm_spy import ( - parse_grgsm_scanner_output, - parse_tshark_output, - arfcn_to_frequency, - validate_band_names, - REGIONAL_BANDS -) - - -class TestParseGrgsmScannerOutput: - """Tests for parse_grgsm_scanner_output().""" - - def test_valid_output_line(self): - """Test parsing a valid grgsm_scanner output line.""" - line = "ARFCN: 23, Freq: 940.6M, CID: 31245, LAC: 1234, MCC: 214, MNC: 01, Pwr: -48" - result = parse_grgsm_scanner_output(line) - - assert result is not None - assert result['type'] == 'tower' - assert result['arfcn'] == 23 - assert result['frequency'] == 940.6 - assert result['cid'] == 31245 - assert result['lac'] == 1234 - assert result['mcc'] == 214 - assert result['mnc'] == 1 - assert result['signal_strength'] == -48.0 - assert 'timestamp' in result - - def test_freq_without_suffix(self): - """Test parsing frequency without M suffix.""" - line = "ARFCN: 975, Freq: 925.2, CID: 13522, LAC: 38722, MCC: 262, MNC: 1, Pwr: -58" - result = parse_grgsm_scanner_output(line) - assert result is not None - assert result['frequency'] == 925.2 - - def test_config_line(self): - """Test that configuration lines are skipped.""" - line = " Configuration: 1 CCCH, not combined" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_neighbour_line(self): - """Test that neighbour cell lines are skipped.""" - line = " Neighbour Cells: 57, 61, 70, 71, 72, 86" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_cell_arfcn_line(self): - """Test that cell ARFCN lines are skipped.""" - line = " Cell ARFCNs: 63, 76" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_progress_line(self): - """Test that progress/status lines are skipped.""" - line = "Scanning GSM900 band..." - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_empty_line(self): - """Test handling of empty lines.""" - result = parse_grgsm_scanner_output("") - assert result is None - - def test_invalid_data(self): - """Test handling of non-numeric values.""" - line = "ARFCN: abc, Freq: xyz, CID: bad, LAC: data, MCC: bad, MNC: bad, Pwr: bad" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_no_identity_filtered(self): - """Test that MCC=0/MNC=0 entries (no network identity) are filtered out.""" - line = "ARFCN: 115, Freq: 925.0M, CID: 0, LAC: 0, MCC: 0, MNC: 0, Pwr: -100" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_mcc_zero_mnc_zero_filtered(self): - """Test that MCC=0/MNC=0 even with valid CID is filtered out.""" - line = "ARFCN: 113, Freq: 924.6M, CID: 1234, LAC: 5678, MCC: 0, MNC: 0, Pwr: -90" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_cid_zero_valid_mcc_passes(self): - """Test that CID=0 with valid MCC/MNC passes (partially decoded cell).""" - line = "ARFCN: 115, Freq: 958.0M, CID: 0, LAC: 21864, MCC: 234, MNC: 10, Pwr: -51" - result = parse_grgsm_scanner_output(line) - assert result is not None - assert result['cid'] == 0 - assert result['mcc'] == 234 - assert result['signal_strength'] == -51.0 - - def test_valid_cid_nonzero(self): - """Test that valid non-zero CID/MCC entries pass through.""" - line = "ARFCN: 115, Freq: 925.0M, CID: 19088, LAC: 21864, MCC: 234, MNC: 10, Pwr: -58" - result = parse_grgsm_scanner_output(line) - assert result is not None - assert result['cid'] == 19088 - assert result['signal_strength'] == -58.0 - - -class TestParseTsharkOutput: - """Tests for parse_tshark_output().""" - - def test_valid_full_output(self): - """Test parsing tshark output with all fields.""" - line = "5\t0xABCD1234\t123456789012345\t1234\t31245" - result = parse_tshark_output(line) - - assert result is not None - assert result['type'] == 'device' - assert result['ta_value'] == 5 - assert result['tmsi'] == '0xABCD1234' - assert result['imsi'] == '123456789012345' - assert result['lac'] == 1234 - assert result['cid'] == 31245 - assert result['distance_meters'] == 5 * 554 # TA * 554 meters - assert 'timestamp' in result - - def test_missing_optional_fields(self): - """Test parsing with missing optional fields (empty tabs). - - A packet with TA but no TMSI/IMSI is discarded since there's - no device identifier to track. - """ - line = "3\t\t\t1234\t31245" - result = parse_tshark_output(line) - assert result is None - - def test_missing_optional_fields_with_tmsi(self): - """Test parsing with TMSI but missing TA, IMSI, CID.""" - line = "\t0xABCD\t\t1234\t" - result = parse_tshark_output(line) - - assert result is not None - assert result['ta_value'] is None - assert result['tmsi'] == '0xABCD' - assert result['imsi'] is None - assert result['lac'] == 1234 - assert result['cid'] is None - - def test_no_ta_value(self): - """Test parsing without TA value (empty first field).""" - line = "\t0xABCD1234\t123456789012345\t1234\t31245" - result = parse_tshark_output(line) - - assert result is not None - assert result['ta_value'] is None - assert result['tmsi'] == '0xABCD1234' - assert result['imsi'] == '123456789012345' - assert result['lac'] == 1234 - assert result['cid'] == 31245 - - def test_invalid_line(self): - """Test handling of invalid tshark output.""" - line = "invalid data" - result = parse_tshark_output(line) - assert result is None - - def test_empty_line(self): - """Test handling of empty lines.""" - result = parse_tshark_output("") - assert result is None - - def test_partial_fields(self): - """Test with fewer than 5 fields.""" - line = "5\t0xABCD1234" # Only 2 fields - result = parse_tshark_output(line) - assert result is None - - -class TestArfcnToFrequency: - """Tests for arfcn_to_frequency().""" - - def test_gsm850_arfcn(self): - """Test ARFCN in GSM850 band.""" - # GSM850: ARFCN 128-251, 869-894 MHz - arfcn = 128 - freq = arfcn_to_frequency(arfcn) - assert freq == 869000000 # 869 MHz - - arfcn = 251 - freq = arfcn_to_frequency(arfcn) - assert freq == 893600000 # 893.6 MHz - - def test_egsm900_arfcn(self): - """Test ARFCN in EGSM900 band.""" - # EGSM900: ARFCN 0-124, DL = 935 + 0.2*ARFCN MHz - arfcn = 0 - freq = arfcn_to_frequency(arfcn) - assert freq == 935000000 # 935.0 MHz - - arfcn = 22 - freq = arfcn_to_frequency(arfcn) - assert freq == 939400000 # 939.4 MHz - - arfcn = 124 - freq = arfcn_to_frequency(arfcn) - assert freq == 959800000 # 959.8 MHz - - def test_egsm900_ext_arfcn(self): - """Test ARFCN in EGSM900 extension band.""" - # EGSM900_EXT: ARFCN 975-1023, DL = 925.2 + 0.2*(ARFCN-975) MHz - arfcn = 975 - freq = arfcn_to_frequency(arfcn) - assert freq == 925200000 # 925.2 MHz - - arfcn = 1023 - freq = arfcn_to_frequency(arfcn) - assert freq == 934800000 # 934.8 MHz - - def test_dcs1800_arfcn(self): - """Test ARFCN in DCS1800 band.""" - # DCS1800: ARFCN 512-885, 1805-1880 MHz - # Note: ARFCN 512 also exists in PCS1900 and will match that first - # Use ARFCN 811+ which is only in DCS1800 - arfcn = 811 # Beyond PCS1900 range (512-810) - freq = arfcn_to_frequency(arfcn) - # 811 is ARFCN offset from 512, so freq = 1805MHz + (811-512)*200kHz - expected = 1805000000 + (811 - 512) * 200000 - assert freq == expected - - arfcn = 885 - freq = arfcn_to_frequency(arfcn) - assert freq == 1879600000 # 1879.6 MHz - - def test_pcs1900_arfcn(self): - """Test ARFCN in PCS1900 band.""" - # PCS1900: ARFCN 512-810, 1930-1990 MHz - # Note: overlaps with DCS1800 ARFCN range, but different frequencies - arfcn = 512 - freq = arfcn_to_frequency(arfcn) - # Will match first band (DCS1800 in Europe config) - assert freq > 0 - - def test_invalid_arfcn(self): - """Test ARFCN outside known ranges.""" - with pytest.raises(ValueError, match="not found in any known GSM band"): - arfcn_to_frequency(9999) - - with pytest.raises(ValueError): - arfcn_to_frequency(-1) - - def test_arfcn_200khz_spacing(self): - """Test that ARFCNs are 200kHz apart.""" - arfcn1 = 128 - arfcn2 = 129 - freq1 = arfcn_to_frequency(arfcn1) - freq2 = arfcn_to_frequency(arfcn2) - assert freq2 - freq1 == 200000 # 200 kHz - - -class TestValidateBandNames: - """Tests for validate_band_names().""" - - def test_valid_americas_bands(self): - """Test valid band names for Americas region.""" - bands = ['GSM850', 'PCS1900'] - result, error = validate_band_names(bands, 'Americas') - assert result == bands - assert error is None - - def test_valid_europe_bands(self): - """Test valid band names for Europe region.""" - # Note: Europe uses EGSM900, not GSM900 - bands = ['EGSM900', 'DCS1800', 'GSM850', 'GSM800'] - result, error = validate_band_names(bands, 'Europe') - assert result == bands - assert error is None - - def test_valid_asia_bands(self): - """Test valid band names for Asia region.""" - # Note: Asia uses EGSM900, not GSM900 - bands = ['EGSM900', 'DCS1800'] - result, error = validate_band_names(bands, 'Asia') - assert result == bands - assert error is None - - def test_invalid_band_for_region(self): - """Test invalid band name for a region.""" - bands = ['GSM900', 'INVALID_BAND'] - result, error = validate_band_names(bands, 'Americas') - assert result == [] - assert error is not None - assert 'Invalid bands' in error - assert 'INVALID_BAND' in error - - def test_invalid_region(self): - """Test invalid region name.""" - bands = ['GSM900'] - result, error = validate_band_names(bands, 'InvalidRegion') - assert result == [] - assert error is not None - assert 'Invalid region' in error - - def test_empty_bands_list(self): - """Test with empty bands list.""" - result, error = validate_band_names([], 'Americas') - assert result == [] - assert error is None - - def test_single_valid_band(self): - """Test with single valid band.""" - bands = ['GSM850'] - result, error = validate_band_names(bands, 'Americas') - assert result == ['GSM850'] - assert error is None - - def test_case_sensitive_band_names(self): - """Test that band names are case-sensitive.""" - bands = ['gsm850'] # lowercase - result, error = validate_band_names(bands, 'Americas') - assert result == [] - assert error is not None - - def test_multiple_invalid_bands(self): - """Test with multiple invalid bands.""" - bands = ['INVALID1', 'GSM850', 'INVALID2'] - result, error = validate_band_names(bands, 'Americas') - assert result == [] - assert error is not None - assert 'INVALID1' in error - assert 'INVALID2' in error - - -class TestRegionalBandsConfig: - """Tests for REGIONAL_BANDS configuration.""" - - def test_all_regions_defined(self): - """Test that all expected regions are defined.""" - assert 'Americas' in REGIONAL_BANDS - assert 'Europe' in REGIONAL_BANDS - assert 'Asia' in REGIONAL_BANDS - - def test_all_bands_have_required_fields(self): - """Test that all bands have required configuration fields.""" - for region, bands in REGIONAL_BANDS.items(): - for band_name, band_config in bands.items(): - assert 'start' in band_config - assert 'end' in band_config - assert 'arfcn_start' in band_config - assert 'arfcn_end' in band_config - - def test_frequency_ranges_valid(self): - """Test that frequency ranges are positive and start < end.""" - for region, bands in REGIONAL_BANDS.items(): - for band_name, band_config in bands.items(): - assert band_config['start'] > 0 - assert band_config['end'] > 0 - assert band_config['start'] < band_config['end'] - - def test_arfcn_ranges_valid(self): - """Test that ARFCN ranges are valid.""" - for region, bands in REGIONAL_BANDS.items(): - for band_name, band_config in bands.items(): - assert band_config['arfcn_start'] >= 0 - assert band_config['arfcn_end'] >= 0 - assert band_config['arfcn_start'] <= band_config['arfcn_end'] diff --git a/utils/constants.py b/utils/constants.py index f51124e..2b5edff 100644 --- a/utils/constants.py +++ b/utils/constants.py @@ -275,13 +275,3 @@ MAX_DEAUTH_ALERTS_AGE_SECONDS = 300 # 5 minutes # Deauth detector sniff timeout (seconds) DEAUTH_SNIFF_TIMEOUT = 0.5 - -# ============================================================================= -# GSM SPY (Cellular Intelligence) -# ============================================================================= - -# Maximum age for GSM tower/device data in DataStore (seconds) -MAX_GSM_AGE_SECONDS = 300 # 5 minutes - -# Timing Advance conversion to meters -GSM_TA_METERS_PER_UNIT = 554 diff --git a/utils/database.py b/utils/database.py index 9e62d87..3211624 100644 --- a/utils/database.py +++ b/utils/database.py @@ -453,134 +453,6 @@ def init_db() -> None: ON tscm_cases(status, created_at) ''') - # ===================================================================== - # GSM (Global System for Mobile) Intelligence Tables - # ===================================================================== - - # gsm_cells - Known cell towers (OpenCellID cache) - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_cells ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - mcc INTEGER NOT NULL, - mnc INTEGER NOT NULL, - lac INTEGER NOT NULL, - cid INTEGER NOT NULL, - lat REAL, - lon REAL, - azimuth INTEGER, - range_meters INTEGER, - samples INTEGER, - radio TEXT, - operator TEXT, - first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - last_verified TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - metadata TEXT, - UNIQUE(mcc, mnc, lac, cid) - ) - ''') - - # gsm_rogues - Detected rogue towers / IMSI catchers - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_rogues ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - arfcn INTEGER NOT NULL, - mcc INTEGER, - mnc INTEGER, - lac INTEGER, - cid INTEGER, - signal_strength REAL, - reason TEXT NOT NULL, - threat_level TEXT DEFAULT 'medium', - detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - location_lat REAL, - location_lon REAL, - acknowledged BOOLEAN DEFAULT 0, - notes TEXT, - metadata TEXT - ) - ''') - - # gsm_signals - 60-day archive of signal observations - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_signals ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - imsi TEXT, - tmsi TEXT, - mcc INTEGER, - mnc INTEGER, - lac INTEGER, - cid INTEGER, - ta_value INTEGER, - signal_strength REAL, - arfcn INTEGER, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - metadata TEXT - ) - ''') - - # gsm_tmsi_log - 24-hour raw pings for crowd density - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_tmsi_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - tmsi TEXT NOT NULL, - lac INTEGER, - cid INTEGER, - ta_value INTEGER, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - ''') - - # gsm_velocity_log - 1-hour buffer for movement tracking - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_velocity_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - device_id TEXT NOT NULL, - prev_ta INTEGER, - curr_ta INTEGER, - prev_cid INTEGER, - curr_cid INTEGER, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - estimated_velocity REAL, - metadata TEXT - ) - ''') - - # GSM indexes for performance - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_cells_location - ON gsm_cells(lat, lon) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_cells_identity - ON gsm_cells(mcc, mnc, lac, cid) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_rogues_severity - ON gsm_rogues(threat_level, detected_at) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_signals_cell_time - ON gsm_signals(cid, lac, timestamp) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_signals_device - ON gsm_signals(imsi, tmsi, timestamp) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_tmsi_log_time - ON gsm_tmsi_log(timestamp) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_velocity_log_device - ON gsm_velocity_log(device_id, timestamp) - ''') - # ===================================================================== # DSC (Digital Selective Calling) Tables # ===================================================================== @@ -2298,60 +2170,3 @@ def cleanup_old_payloads(max_age_hours: int = 24) -> int: ''', (f'-{max_age_hours} hours',)) return cursor.rowcount - -# ============================================================================= -# GSM Cleanup Functions -# ============================================================================= - -def cleanup_old_gsm_signals(max_age_days: int = 60) -> int: - """ - Remove old GSM signal observations (60-day archive). - - Args: - max_age_days: Maximum age in days (default: 60) - - Returns: - Number of deleted entries - """ - with get_db() as conn: - cursor = conn.execute(''' - DELETE FROM gsm_signals - WHERE timestamp < datetime('now', ?) - ''', (f'-{max_age_days} days',)) - return cursor.rowcount - - -def cleanup_old_gsm_tmsi_log(max_age_hours: int = 24) -> int: - """ - Remove old TMSI log entries (24-hour buffer for crowd density). - - Args: - max_age_hours: Maximum age in hours (default: 24) - - Returns: - Number of deleted entries - """ - with get_db() as conn: - cursor = conn.execute(''' - DELETE FROM gsm_tmsi_log - WHERE timestamp < datetime('now', ?) - ''', (f'-{max_age_hours} hours',)) - return cursor.rowcount - - -def cleanup_old_gsm_velocity_log(max_age_hours: int = 1) -> int: - """ - Remove old velocity log entries (1-hour buffer for movement tracking). - - Args: - max_age_hours: Maximum age in hours (default: 1) - - Returns: - Number of deleted entries - """ - with get_db() as conn: - cursor = conn.execute(''' - DELETE FROM gsm_velocity_log - WHERE timestamp < datetime('now', ?) - ''', (f'-{max_age_hours} hours',)) - return cursor.rowcount diff --git a/utils/dependencies.py b/utils/dependencies.py index e6a1bee..a12eca7 100644 --- a/utils/dependencies.py +++ b/utils/dependencies.py @@ -444,38 +444,6 @@ TOOL_DEPENDENCIES = { } } }, - 'gsm': { - 'name': 'GSM Intelligence', - 'tools': { - 'grgsm_scanner': { - 'required': True, - 'description': 'gr-gsm scanner for finding GSM towers', - 'install': { - 'apt': 'Build gr-gsm from source: https://github.com/bkerler/gr-gsm', - 'brew': 'brew install gr-gsm (may require manual build)', - 'manual': 'https://github.com/bkerler/gr-gsm' - } - }, - 'grgsm_livemon': { - 'required': True, - 'description': 'gr-gsm live monitor for decoding GSM signals', - 'install': { - 'apt': 'Included with gr-gsm package', - 'brew': 'Included with gr-gsm', - 'manual': 'Included with gr-gsm' - } - }, - 'tshark': { - 'required': True, - 'description': 'Wireshark CLI for parsing GSM packets', - 'install': { - 'apt': 'sudo apt-get install tshark', - 'brew': 'brew install wireshark', - 'manual': 'https://www.wireshark.org/download.html' - } - } - } - } } diff --git a/utils/gsm_geocoding.py b/utils/gsm_geocoding.py deleted file mode 100644 index feaf164..0000000 --- a/utils/gsm_geocoding.py +++ /dev/null @@ -1,226 +0,0 @@ -"""GSM Cell Tower Geocoding Service. - -Provides hybrid cache-first geocoding with async API fallback for cell towers. -""" - -from __future__ import annotations - -import logging -import queue -from typing import Any - -import requests - -import config -from utils.database import get_db - -logger = logging.getLogger('intercept.gsm_geocoding') - -# Queue for pending geocoding requests -_geocoding_queue = queue.Queue(maxsize=100) - - -def lookup_cell_coordinates(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None: - """ - Lookup cell tower coordinates with cache-first strategy. - - Strategy: - 1. Check gsm_cells table (cache) - fast synchronous lookup - 2. If not found, return None (caller decides whether to use API) - - Args: - mcc: Mobile Country Code - mnc: Mobile Network Code - lac: Location Area Code - cid: Cell ID - - Returns: - dict with keys: lat, lon, source='cache', azimuth (optional), - range_meters (optional), operator (optional), radio (optional) - Returns None if not found in cache. - """ - try: - with get_db() as conn: - result = conn.execute(''' - SELECT lat, lon, azimuth, range_meters, operator, radio - FROM gsm_cells - WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? - ''', (mcc, mnc, lac, cid)).fetchone() - - if result and result['lat'] is not None and result['lon'] is not None: - return { - 'lat': result['lat'], - 'lon': result['lon'], - 'source': 'cache', - 'azimuth': result['azimuth'], - 'range_meters': result['range_meters'], - 'operator': result['operator'], - 'radio': result['radio'] - } - - return None - - except Exception as e: - logger.error(f"Error looking up coordinates from cache: {e}") - return None - - -def _get_api_key() -> str: - """Get OpenCellID API key at runtime (env var first, then database).""" - env_key = config.GSM_OPENCELLID_API_KEY - if env_key: - return env_key - from utils.database import get_setting - return get_setting('gsm.opencellid.api_key', '') - - -def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None: - """ - Lookup cell tower from OpenCellID API and cache result. - - Args: - mcc: Mobile Country Code - mnc: Mobile Network Code - lac: Location Area Code - cid: Cell ID - - Returns: - dict with keys: lat, lon, source='api', azimuth (optional), - range_meters (optional), operator (optional), radio (optional) - Returns None if API call fails or cell not found. - """ - try: - api_key = _get_api_key() - if not api_key: - logger.warning("OpenCellID API key not configured") - return None - - api_url = config.GSM_OPENCELLID_API_URL - params = { - 'key': api_key, - 'mcc': mcc, - 'mnc': mnc, - 'lac': lac, - 'cellid': cid, - 'format': 'json' - } - - response = requests.get(api_url, params=params, timeout=10) - - if response.status_code == 200: - cell_data = response.json() - lat = cell_data.get('lat') - lon = cell_data.get('lon') - - # Validate response has actual coordinates - if lat is None or lon is None: - logger.warning( - f"OpenCellID API returned 200 but no coordinates for " - f"MCC={mcc} MNC={mnc} LAC={lac} CID={cid}: {cell_data}" - ) - return None - - # Cache the result - with get_db() as conn: - conn.execute(''' - INSERT OR REPLACE INTO gsm_cells - (mcc, mnc, lac, cid, lat, lon, azimuth, range_meters, samples, radio, operator, last_verified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) - ''', ( - mcc, mnc, lac, cid, - lat, lon, - cell_data.get('azimuth'), - cell_data.get('range'), - cell_data.get('samples'), - cell_data.get('radio'), - cell_data.get('operator') - )) - conn.commit() - - logger.info(f"Cached cell tower from API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid} -> ({lat}, {lon})") - - return { - 'lat': lat, - 'lon': lon, - 'source': 'api', - 'azimuth': cell_data.get('azimuth'), - 'range_meters': cell_data.get('range'), - 'operator': cell_data.get('operator'), - 'radio': cell_data.get('radio') - } - else: - logger.warning( - f"OpenCellID API returned {response.status_code} for " - f"MCC={mcc} MNC={mnc} LAC={lac} CID={cid}: {response.text[:200]}" - ) - return None - - except Exception as e: - logger.error(f"Error calling OpenCellID API: {e}") - return None - - -def enrich_tower_data(tower_data: dict[str, Any]) -> dict[str, Any]: - """ - Enrich tower data with coordinates using cache-first strategy. - - If coordinates found in cache, adds them immediately. - If not found, marks as 'pending' and queues for background API lookup. - - Args: - tower_data: Dictionary with keys mcc, mnc, lac, cid (and other tower data) - - Returns: - Enriched tower_data dict with added fields: - - lat, lon (if found in cache) - - status='pending' (if needs API lookup) - - source='cache' (if from cache) - """ - mcc = tower_data.get('mcc') - mnc = tower_data.get('mnc') - lac = tower_data.get('lac') - cid = tower_data.get('cid') - - # Validate required fields - if not all([mcc is not None, mnc is not None, lac is not None, cid is not None]): - logger.warning(f"Tower data missing required fields: {tower_data}") - return tower_data - - # Try cache lookup - coords = lookup_cell_coordinates(mcc, mnc, lac, cid) - - if coords: - # Found in cache - add coordinates immediately - tower_data['lat'] = coords['lat'] - tower_data['lon'] = coords['lon'] - tower_data['source'] = 'cache' - - # Add optional fields if available - if coords.get('azimuth') is not None: - tower_data['azimuth'] = coords['azimuth'] - if coords.get('range_meters') is not None: - tower_data['range_meters'] = coords['range_meters'] - if coords.get('operator'): - tower_data['operator'] = coords['operator'] - if coords.get('radio'): - tower_data['radio'] = coords['radio'] - - logger.debug(f"Cache hit for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - else: - # Not in cache - mark as pending and queue for API lookup - tower_data['status'] = 'pending' - tower_data['source'] = 'unknown' - - # Queue for background geocoding (non-blocking) - try: - _geocoding_queue.put_nowait(tower_data.copy()) - logger.debug(f"Queued tower for geocoding: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - except queue.Full: - logger.warning("Geocoding queue full, dropping tower") - - return tower_data - - -def get_geocoding_queue() -> queue.Queue: - """Get the geocoding queue for the background worker.""" - return _geocoding_queue diff --git a/utils/logging.py b/utils/logging.py index addbabe..3d2cc6a 100644 --- a/utils/logging.py +++ b/utils/logging.py @@ -28,4 +28,3 @@ wifi_logger = get_logger('intercept.wifi') bluetooth_logger = get_logger('intercept.bluetooth') adsb_logger = get_logger('intercept.adsb') satellite_logger = get_logger('intercept.satellite') -gsm_spy_logger = get_logger('intercept.gsm_spy')