From 82f442ffb852113017eebcf02c57dc63b5b991f1 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 18:37:48 +0000 Subject: [PATCH 01/24] Fix tshark capture: add GSMTAP filter, line buffering, stderr capture - Add capture filter (-f 'udp port 4729') to only capture GSMTAP packets - Add -l flag for line-buffered output on live capture - Add early exit detection for tshark with stderr capture - Add stderr reader thread in monitor_thread for ongoing tshark diagnostics - Clean up grgsm_livemon if tshark fails to start Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index b1b5607..1634417 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -313,7 +313,10 @@ def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subproce raise FileNotFoundError('tshark not found. Please install wireshark/tshark.') tshark_cmd = [ - 'tshark', '-i', 'lo', + 'tshark', + '-i', 'lo', + '-l', # Line-buffered output for live capture + '-f', 'udp port 4729', # Capture filter: only GSMTAP packets '-Y', 'gsm_a.rr.timing_advance || gsm_a.tmsi || gsm_a.imsi', '-T', 'fields', '-e', 'gsm_a.rr.timing_advance', @@ -333,6 +336,22 @@ def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subproce register_process(tshark_proc) logger.info(f"Started tshark (PID: {tshark_proc.pid})") + # Check tshark didn't exit immediately + time.sleep(1) + if tshark_proc.poll() is not None: + stderr_output = '' + try: + stderr_output = tshark_proc.stderr.read() + except Exception: + pass + exit_code = tshark_proc.returncode + logger.error(f"tshark exited immediately (code: {exit_code}). stderr: {stderr_output[:500]}") + # Clean up grgsm_livemon since monitoring can't work without tshark + safe_terminate(grgsm_proc) + unregister_process(grgsm_proc) + unregister_process(tshark_proc) + raise RuntimeError(f'tshark failed (exit code {exit_code}): {stderr_output[:200]}') + return grgsm_proc, tshark_proc @@ -1596,8 +1615,18 @@ def monitor_thread(process): finally: output_queue_local.put(('eof', None)) + def read_stderr(): + try: + for line in iter(process.stderr.readline, ''): + if line: + logger.debug(f"tshark stderr: {line.strip()}") + except Exception: + pass + stdout_thread = threading.Thread(target=read_stdout, daemon=True) stdout_thread.start() + stderr_thread = threading.Thread(target=read_stderr, daemon=True) + stderr_thread.start() try: while app_module.gsm_spy_monitor_process: From 6b7f817aa64abd1be613636bfeaaf9b4886ea9da Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 18:41:30 +0000 Subject: [PATCH 02/24] Add live monitoring status overlay with heartbeat updates Backend: monitor_thread sends periodic monitor_heartbeat events (every 5s) with elapsed time, packet count, and device count so the frontend knows monitoring is active. Frontend: new monitoring overlay replaces scan progress bar when auto-monitor starts. Shows pulsing green indicator, ARFCN being monitored, live elapsed timer, packet/device counts, and "Listening..."/"Capturing" activity state. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 21 ++++ templates/gsm_spy_dashboard.html | 159 ++++++++++++++++++++++++++++++- 2 files changed, 179 insertions(+), 1 deletion(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index 1634417..fea0834 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -1628,6 +1628,10 @@ def monitor_thread(process): stderr_thread = threading.Thread(target=read_stderr, daemon=True) stderr_thread.start() + monitor_start_time = time.time() + packets_captured = 0 + last_heartbeat = time.time() + try: while app_module.gsm_spy_monitor_process: # Check if process died @@ -1635,6 +1639,21 @@ def monitor_thread(process): logger.info(f"Monitor process exited (code: {process.returncode})") break + # Send periodic heartbeat so frontend knows monitor is alive + now = time.time() + if now - last_heartbeat >= 5: + last_heartbeat = now + elapsed = int(now - monitor_start_time) + try: + app_module.gsm_spy_queue.put_nowait({ + 'type': 'monitor_heartbeat', + 'elapsed': elapsed, + 'packets': packets_captured, + 'devices': len(app_module.gsm_spy_devices) + }) + except queue.Full: + pass + # Get output from queue with timeout try: msg_type, line = output_queue_local.get(timeout=1.0) @@ -1646,6 +1665,8 @@ def monitor_thread(process): parsed = parse_tshark_output(line) if parsed: + packets_captured += 1 + # Store in DataStore key = parsed.get('tmsi') or parsed.get('imsi') or str(time.time()) app_module.gsm_spy_devices[key] = parsed diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html index dc673ad..ff05a7c 100644 --- a/templates/gsm_spy_dashboard.html +++ b/templates/gsm_spy_dashboard.html @@ -525,6 +525,91 @@ box-shadow: 0 0 8px rgba(0, 229, 255, 0.4); } + /* Monitor Status Overlay */ + .monitor-status-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + z-index: 1000; + background: linear-gradient(180deg, rgba(10, 14, 20, 0.95) 0%, rgba(10, 14, 20, 0.85) 100%); + border-bottom: 1px solid var(--accent-green, #4caf50); + backdrop-filter: blur(8px); + padding: 10px 16px; + } + + .monitor-status-inner { + max-width: 600px; + margin: 0 auto; + } + + .monitor-status-row { + display: flex; + align-items: center; + gap: 10px; + font-family: var(--font-mono); + font-size: 12px; + } + + .monitor-pulse { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--accent-green, #4caf50); + box-shadow: 0 0 6px var(--accent-green, #4caf50); + animation: pulse-glow 2s ease-in-out infinite; + } + + @keyframes pulse-glow { + 0%, 100% { opacity: 1; box-shadow: 0 0 6px var(--accent-green, #4caf50); } + 50% { opacity: 0.5; box-shadow: 0 0 12px var(--accent-green, #4caf50); } + } + + .monitor-label { + color: var(--accent-green, #4caf50); + font-weight: 700; + letter-spacing: 1.5px; + font-size: 11px; + } + + .monitor-arfcn { + color: var(--text-primary); + font-weight: 600; + } + + .monitor-elapsed { + margin-left: auto; + color: var(--text-secondary); + } + + .monitor-stats-row { + display: flex; + align-items: center; + gap: 8px; + margin-top: 5px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-secondary); + padding-left: 18px; + } + + .monitor-stat-sep { + width: 3px; + height: 3px; + border-radius: 50%; + background: var(--text-secondary); + opacity: 0.5; + } + + .monitor-listening { + animation: listening-fade 2.5s ease-in-out infinite; + } + + @keyframes listening-fade { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.3; } + } + /* Right Sidebar */ .right-sidebar { background: var(--bg-panel); @@ -1293,6 +1378,23 @@ +
@@ -1667,6 +1769,7 @@ eventSource = null; } document.getElementById('scanProgress').style.display = 'none'; + hideMonitorStatus(); console.log('[GSM SPY] Scanner stopped'); }) .catch(error => { @@ -1740,13 +1843,16 @@ updateScanStatus('Scan #' + data.scan + ' complete (' + data.towers_found + ' towers, ' + data.duration + 's)'); document.getElementById('scanProgressBar').style.width = '100%'; } else if (data.type === 'auto_monitor_started') { - updateScanStatus('Monitoring ARFCN ' + data.arfcn + ' for devices...'); + showMonitorStatus(data.arfcn); console.log('[GSM SPY] Auto-monitor started on ARFCN', data.arfcn); + } else if (data.type === 'monitor_heartbeat') { + updateMonitorStatus(data.elapsed, data.packets, data.devices); } else if (data.type === 'error') { console.error('[GSM SPY] Server error:', data.message); updateScanStatus('Error: ' + data.message); } else if (data.type === 'disconnected') { console.warn('[GSM SPY] Server disconnected stream'); + hideMonitorStatus(); } } catch (error) { console.error('[GSM SPY] Error parsing event:', error, 'raw:', e.data); @@ -2002,6 +2108,57 @@ statusText.textContent = message; } + let monitorStartTime = null; + let monitorTimerInterval = null; + + function showMonitorStatus(arfcn) { + // Hide scan progress, show monitor status + document.getElementById('scanProgress').style.display = 'none'; + const overlay = document.getElementById('monitorStatus'); + overlay.style.display = 'block'; + document.getElementById('monitorArfcn').textContent = 'ARFCN ' + arfcn; + document.getElementById('monitorPackets').textContent = '0'; + document.getElementById('monitorDevices').textContent = '0'; + document.getElementById('monitorActivity').textContent = 'Listening...'; + + // Start local elapsed timer for smooth updates between heartbeats + monitorStartTime = Date.now(); + if (monitorTimerInterval) clearInterval(monitorTimerInterval); + monitorTimerInterval = setInterval(function() { + const elapsed = Math.floor((Date.now() - monitorStartTime) / 1000); + document.getElementById('monitorElapsed').textContent = formatElapsed(elapsed); + }, 1000); + } + + function updateMonitorStatus(elapsed, packets, devices) { + const overlay = document.getElementById('monitorStatus'); + if (overlay.style.display === 'none') return; + document.getElementById('monitorElapsed').textContent = formatElapsed(elapsed); + document.getElementById('monitorPackets').textContent = packets; + document.getElementById('monitorDevices').textContent = devices; + // Sync local timer with server elapsed + monitorStartTime = Date.now() - (elapsed * 1000); + // Flash activity indicator on heartbeat + const activity = document.getElementById('monitorActivity'); + activity.textContent = packets > 0 ? 'Capturing' : 'Listening...'; + activity.style.color = packets > 0 ? 'var(--accent-green, #4caf50)' : ''; + } + + function hideMonitorStatus() { + document.getElementById('monitorStatus').style.display = 'none'; + if (monitorTimerInterval) { + clearInterval(monitorTimerInterval); + monitorTimerInterval = null; + } + monitorStartTime = null; + } + + function formatElapsed(seconds) { + const m = Math.floor(seconds / 60); + const s = seconds % 60; + return String(m).padStart(2, '0') + ':' + String(s).padStart(2, '0'); + } + function updateTowersList() { const listDiv = document.getElementById('towersList'); const towerCount = Object.keys(towers).length; From 87782319f2a62a8d0e25a73f0710dd44200caf8b Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 18:49:20 +0000 Subject: [PATCH 03/24] Auto-discover tshark field names for GSM protocol compatibility tshark field names differ between Wireshark versions (3.x vs 4.x): - 3.x: gsm_a.rr.timing_advance, gsm_a.tmsi, gsm_a.cellid - 4.x: gsm_a_rr.timing_adv, gsm_a_dtap.tmsi, e212.ci Added _discover_tshark_fields() that queries `tshark -G fields` to find which field names are available on the installed version, then uses the correct ones for the capture filter and field extraction. Results are cached after first discovery. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 92 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index fea0834..4540382 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -238,6 +238,84 @@ def validate_band_names(bands: list[str], region: str) -> tuple[list[str], str | return bands, None +# tshark field name discovery - field names vary between Wireshark versions +_tshark_fields_cache: dict[str, str] | None = None + + +def _discover_tshark_fields() -> dict[str, str]: + """Discover correct tshark field names for GSM A protocol. + + Different Wireshark versions use different field names: + - Wireshark 3.x: gsm_a.rr.timing_advance, gsm_a.tmsi, gsm_a.imsi, gsm_a.lac, gsm_a.cellid + - Wireshark 4.x: gsm_a_rr.timing_adv, gsm_a_dtap.tmsi, e212.imsi, e212.lac, e212.ci + + Returns: + Dict mapping logical names to actual tshark field names: + {'ta': ..., 'tmsi': ..., 'imsi': ..., 'lac': ..., 'cid': ...} + """ + global _tshark_fields_cache + if _tshark_fields_cache is not None: + return _tshark_fields_cache + + # Candidate field names for each logical field, in preference order + candidates = { + 'ta': [ + 'gsm_a.rr.timing_advance', + 'gsm_a_rr.timing_adv', + 'gsm_a_rr.timing_advance', + ], + 'tmsi': [ + 'gsm_a.tmsi', + 'gsm_a_dtap.tmsi', + 'gsm_a.dtap.tmsi', + ], + 'imsi': [ + 'gsm_a.imsi', + 'e212.imsi', + 'gsm_a_dtap.imsi', + ], + 'lac': [ + 'gsm_a.lac', + 'e212.lac', + 'gsm_a_bssmap.lac', + ], + 'cid': [ + 'gsm_a.cellid', + 'e212.ci', + 'gsm_a_bssmap.cell_ci', + 'gsm_a.cell_ci', + ], + } + + # Query tshark for all available GSM fields + available_fields = set() + try: + result = subprocess.run( + ['tshark', '-G', 'fields'], + capture_output=True, text=True, timeout=10 + ) + for line in result.stdout.splitlines(): + parts = line.split('\t') + if len(parts) >= 3: + available_fields.add(parts[2]) # Field name is 3rd column + except Exception as e: + logger.warning(f"Could not query tshark fields: {e}") + + # Match each logical field to the first available candidate + resolved = {} + for logical_name, field_candidates in candidates.items(): + resolved[logical_name] = field_candidates[0] # default fallback + if available_fields: + for candidate in field_candidates: + if candidate in available_fields: + resolved[logical_name] = candidate + break + + logger.info(f"Discovered tshark fields: {resolved}") + _tshark_fields_cache = resolved + return resolved + + def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subprocess.Popen, subprocess.Popen]: """Start grgsm_livemon and tshark processes for monitoring an ARFCN. @@ -312,18 +390,20 @@ def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subproce unregister_process(grgsm_proc) raise FileNotFoundError('tshark not found. Please install wireshark/tshark.') + fields = _discover_tshark_fields() + display_filter = f"{fields['ta']} || {fields['tmsi']} || {fields['imsi']}" tshark_cmd = [ 'tshark', '-i', 'lo', '-l', # Line-buffered output for live capture '-f', 'udp port 4729', # Capture filter: only GSMTAP packets - '-Y', 'gsm_a.rr.timing_advance || gsm_a.tmsi || gsm_a.imsi', + '-Y', display_filter, '-T', 'fields', - '-e', 'gsm_a.rr.timing_advance', - '-e', 'gsm_a.tmsi', - '-e', 'gsm_a.imsi', - '-e', 'gsm_a.lac', - '-e', 'gsm_a.cellid' + '-e', fields['ta'], + '-e', fields['tmsi'], + '-e', fields['imsi'], + '-e', fields['lac'], + '-e', fields['cid'] ] logger.info(f"Starting tshark: {' '.join(tshark_cmd)}") tshark_proc = subprocess.Popen( From 182e1f32399d2c22974b09fc1b43f288cf01befe Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 18:56:50 +0000 Subject: [PATCH 04/24] Fix tshark field discovery to validate with actual extraction test tshark -G fields lists fields that exist in the protocol tree but aren't all valid for -T fields -e extraction. Changed discovery to actually test candidates by running tshark -T fields -e -r /dev/null and parsing stderr for invalid field names. This correctly identifies which fields work for extraction on the installed version. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 74 +++++++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index 4540382..9197d71 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -245,9 +245,9 @@ _tshark_fields_cache: dict[str, str] | None = None def _discover_tshark_fields() -> dict[str, str]: """Discover correct tshark field names for GSM A protocol. - Different Wireshark versions use different field names: - - Wireshark 3.x: gsm_a.rr.timing_advance, gsm_a.tmsi, gsm_a.imsi, gsm_a.lac, gsm_a.cellid - - Wireshark 4.x: gsm_a_rr.timing_adv, gsm_a_dtap.tmsi, e212.imsi, e212.lac, e212.ci + Different Wireshark versions use different field names for -e extraction. + We validate candidates by actually testing them with tshark, since + `tshark -G fields` can list fields that aren't valid for -T fields -e. Returns: Dict mapping logical names to actual tshark field names: @@ -270,8 +270,8 @@ def _discover_tshark_fields() -> dict[str, str]: 'gsm_a.dtap.tmsi', ], 'imsi': [ - 'gsm_a.imsi', 'e212.imsi', + 'gsm_a.imsi', 'gsm_a_dtap.imsi', ], 'lac': [ @@ -287,31 +287,55 @@ def _discover_tshark_fields() -> dict[str, str]: ], } - # Query tshark for all available GSM fields - available_fields = set() - try: - result = subprocess.run( - ['tshark', '-G', 'fields'], - capture_output=True, text=True, timeout=10 - ) - for line in result.stdout.splitlines(): - parts = line.split('\t') - if len(parts) >= 3: - available_fields.add(parts[2]) # Field name is 3rd column - except Exception as e: - logger.warning(f"Could not query tshark fields: {e}") + def _test_fields(field_list: list[str]) -> set[str]: + """Test which fields tshark accepts for -e extraction.""" + cmd = ['tshark', '-T', 'fields'] + for f in field_list: + cmd.extend(['-e', f]) + cmd.extend(['-r', '/dev/null']) + try: + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=10 + ) + if result.returncode == 0: + return set(field_list) + # Parse stderr for invalid fields + invalid = set() + for line in result.stderr.splitlines(): + line = line.strip() + # tshark lists invalid fields one per line after the error header + for f in field_list: + if f in line: + invalid.add(f) + return set(field_list) - invalid + except Exception as e: + logger.warning(f"Could not validate tshark fields: {e}") + return set() - # Match each logical field to the first available candidate + # Collect all unique candidate field names and test them in one call + all_candidates = [] + for field_list in candidates.values(): + all_candidates.extend(field_list) + valid_fields = _test_fields(list(set(all_candidates))) + logger.info(f"Valid tshark -e fields: {sorted(valid_fields)}") + + # Match each logical field to the first valid candidate resolved = {} for logical_name, field_candidates in candidates.items(): - resolved[logical_name] = field_candidates[0] # default fallback - if available_fields: - for candidate in field_candidates: - if candidate in available_fields: - resolved[logical_name] = candidate - break + resolved[logical_name] = None + for candidate in field_candidates: + if candidate in valid_fields: + resolved[logical_name] = candidate + break + if resolved[logical_name] is None: + # No candidate was valid - use first as fallback + resolved[logical_name] = field_candidates[0] + logger.warning( + f"No valid tshark field found for '{logical_name}', " + f"using fallback: {field_candidates[0]}" + ) - logger.info(f"Discovered tshark fields: {resolved}") + logger.info(f"Resolved tshark fields: {resolved}") _tshark_fields_cache = resolved return resolved From 4d7be047daba62eb28c4445989b639f1c2187b13 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 19:14:09 +0000 Subject: [PATCH 05/24] Fix tshark crash by skipping invalid fields instead of using fallbacks When tshark field discovery finds no valid candidate for a logical field (e.g. timing_advance, cellid), the old code fell back to the first candidate name even though it was known to be invalid. This caused tshark to exit immediately with "Some fields aren't valid". Now fields resolve to None when no valid candidate exists, and the tshark command is built using only validated fields. The parser dynamically maps columns via field_order instead of assuming a fixed 5-column layout. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 265 ++++++++++++++++++++++++++++------------------ 1 file changed, 163 insertions(+), 102 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index 9197d71..a171e16 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -242,97 +242,117 @@ def validate_band_names(bands: list[str], region: str) -> tuple[list[str], str | _tshark_fields_cache: dict[str, str] | None = None -def _discover_tshark_fields() -> dict[str, str]: +def _discover_tshark_fields() -> dict[str, str | None]: """Discover correct tshark field names for GSM A protocol. - Different Wireshark versions use different field names for -e extraction. - We validate candidates by actually testing them with tshark, since - `tshark -G fields` can list fields that aren't valid for -T fields -e. + Searches tshark's registered fields for keywords to find the actual + names used by the installed Wireshark version, then validates them. Returns: - Dict mapping logical names to actual tshark field names: + Dict mapping logical names to actual tshark field names (or None): {'ta': ..., 'tmsi': ..., 'imsi': ..., 'lac': ..., 'cid': ...} """ global _tshark_fields_cache if _tshark_fields_cache is not None: return _tshark_fields_cache - # Candidate field names for each logical field, in preference order - candidates = { - 'ta': [ - 'gsm_a.rr.timing_advance', - 'gsm_a_rr.timing_adv', - 'gsm_a_rr.timing_advance', - ], - 'tmsi': [ - 'gsm_a.tmsi', - 'gsm_a_dtap.tmsi', - 'gsm_a.dtap.tmsi', - ], - 'imsi': [ - 'e212.imsi', - 'gsm_a.imsi', - 'gsm_a_dtap.imsi', - ], - 'lac': [ - 'gsm_a.lac', - 'e212.lac', - 'gsm_a_bssmap.lac', - ], - 'cid': [ - 'gsm_a.cellid', - 'e212.ci', - 'gsm_a_bssmap.cell_ci', - 'gsm_a.cell_ci', - ], + # Search patterns for each logical field (applied to tshark -G fields output) + # Each entry: (keyword_patterns, exclusion_patterns) + # We search the field filter name column for matches + search_config = { + 'ta': { + 'keywords': ['timing_adv', 'timing_advance'], + 'prefer': ['gsm_a'], # prefer GSM A protocol fields + }, + 'tmsi': { + 'keywords': ['.tmsi'], + 'prefer': ['gsm_a'], + }, + 'imsi': { + 'keywords': ['.imsi'], + 'prefer': ['e212', 'gsm_a'], + }, + 'lac': { + 'keywords': ['.lac'], + 'prefer': ['gsm_a', 'e212'], + }, + 'cid': { + 'keywords': ['cellid', 'cell_ci', '.cell_id', 'e212.ci'], + 'prefer': ['gsm_a', 'e212'], + }, } - def _test_fields(field_list: list[str]) -> set[str]: - """Test which fields tshark accepts for -e extraction.""" - cmd = ['tshark', '-T', 'fields'] - for f in field_list: - cmd.extend(['-e', f]) - cmd.extend(['-r', '/dev/null']) - try: - result = subprocess.run( - cmd, capture_output=True, text=True, timeout=10 - ) - if result.returncode == 0: - return set(field_list) - # Parse stderr for invalid fields - invalid = set() - for line in result.stderr.splitlines(): - line = line.strip() - # tshark lists invalid fields one per line after the error header - for f in field_list: - if f in line: - invalid.add(f) - return set(field_list) - invalid - except Exception as e: - logger.warning(f"Could not validate tshark fields: {e}") - return set() + # Step 1: Get all field names from tshark (F lines only, not P protocol lines) + all_field_names = [] + try: + result = subprocess.run( + ['tshark', '-G', 'fields'], + capture_output=True, text=True, timeout=15 + ) + for line in result.stdout.splitlines(): + if not line.startswith('F\t'): + continue # Only actual fields, not protocols + parts = line.split('\t') + if len(parts) >= 3: + all_field_names.append(parts[2]) + except Exception as e: + logger.warning(f"Could not query tshark fields: {e}") - # Collect all unique candidate field names and test them in one call - all_candidates = [] + # Step 2: Search for candidate fields matching each logical name + candidates: dict[str, list[str]] = {} + for logical_name, config in search_config.items(): + matches = [] + for field_name in all_field_names: + for keyword in config['keywords']: + if keyword in field_name: + matches.append(field_name) + break + # Sort: preferred protocol prefixes first + def sort_key(name): + for i, pref in enumerate(config['prefer']): + if name.startswith(pref): + return i + return 100 + matches.sort(key=sort_key) + candidates[logical_name] = matches + + logger.info(f"tshark field candidates from -G fields: {candidates}") + + # Step 3: Validate candidates by testing with tshark -r /dev/null + # Collect all unique candidate names + all_candidate_names = set() for field_list in candidates.values(): - all_candidates.extend(field_list) - valid_fields = _test_fields(list(set(all_candidates))) - logger.info(f"Valid tshark -e fields: {sorted(valid_fields)}") + all_candidate_names.update(field_list) - # Match each logical field to the first valid candidate - resolved = {} + valid_fields = set() + if all_candidate_names: + # Test in batches to identify which are valid + # Test each individually since batch testing makes it hard to identify valid ones + for field_name in all_candidate_names: + try: + result = subprocess.run( + ['tshark', '-T', 'fields', '-e', field_name, '-r', '/dev/null'], + capture_output=True, text=True, timeout=5 + ) + if result.returncode == 0 or 'aren\'t valid' not in result.stderr: + valid_fields.add(field_name) + except Exception: + pass + + logger.info(f"Validated tshark -e fields: {sorted(valid_fields)}") + + # Step 4: Resolve each logical field to the first valid candidate + resolved: dict[str, str | None] = {} for logical_name, field_candidates in candidates.items(): resolved[logical_name] = None for candidate in field_candidates: if candidate in valid_fields: resolved[logical_name] = candidate break - if resolved[logical_name] is None: - # No candidate was valid - use first as fallback - resolved[logical_name] = field_candidates[0] + if resolved[logical_name] is None and field_candidates: logger.warning( - f"No valid tshark field found for '{logical_name}', " - f"using fallback: {field_candidates[0]}" + f"No valid tshark field for '{logical_name}'. " + f"Candidates were: {field_candidates}" ) logger.info(f"Resolved tshark fields: {resolved}") @@ -340,15 +360,16 @@ def _discover_tshark_fields() -> dict[str, str]: return resolved -def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subprocess.Popen, subprocess.Popen]: +def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subprocess.Popen, subprocess.Popen, list[str]]: """Start grgsm_livemon and tshark processes for monitoring an ARFCN. Returns: - Tuple of (grgsm_process, tshark_process) + Tuple of (grgsm_process, tshark_process, field_order) + field_order is the list of logical field names in tshark column order. Raises: FileNotFoundError: If grgsm_livemon or tshark not found - RuntimeError: If grgsm_livemon exits immediately + RuntimeError: If grgsm_livemon or tshark exits immediately """ frequency_hz = arfcn_to_frequency(arfcn) frequency_mhz = frequency_hz / 1e6 @@ -415,20 +436,39 @@ def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subproce raise FileNotFoundError('tshark not found. Please install wireshark/tshark.') fields = _discover_tshark_fields() - display_filter = f"{fields['ta']} || {fields['tmsi']} || {fields['imsi']}" + + # Build field list from only valid (non-None) fields + # Track order so parser knows which column is which + field_order = [] # list of logical names in column order tshark_cmd = [ 'tshark', '-i', 'lo', '-l', # Line-buffered output for live capture '-f', 'udp port 4729', # Capture filter: only GSMTAP packets - '-Y', display_filter, - '-T', 'fields', - '-e', fields['ta'], - '-e', fields['tmsi'], - '-e', fields['imsi'], - '-e', fields['lac'], - '-e', fields['cid'] ] + + # Build display filter from available fields + filter_parts = [] + for logical_name in ['ta', 'tmsi', 'imsi']: + if fields.get(logical_name): + filter_parts.append(fields[logical_name]) + if filter_parts: + tshark_cmd.extend(['-Y', ' || '.join(filter_parts)]) + + tshark_cmd.extend(['-T', 'fields']) + + # Add -e for each available field in known order + for logical_name in ['ta', 'tmsi', 'imsi', 'lac', 'cid']: + if fields.get(logical_name): + tshark_cmd.extend(['-e', fields[logical_name]]) + field_order.append(logical_name) + + if not field_order: + safe_terminate(grgsm_proc) + unregister_process(grgsm_proc) + raise RuntimeError('No valid tshark fields found for GSM capture') + + logger.info(f"tshark field order: {field_order}") logger.info(f"Starting tshark: {' '.join(tshark_cmd)}") tshark_proc = subprocess.Popen( tshark_cmd, @@ -456,7 +496,7 @@ def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subproce unregister_process(tshark_proc) raise RuntimeError(f'tshark failed (exit code {exit_code}): {stderr_output[:200]}') - return grgsm_proc, tshark_proc + return grgsm_proc, tshark_proc, field_order def _start_and_register_monitor(arfcn: int, device_index: int) -> None: @@ -470,7 +510,7 @@ def _start_and_register_monitor(arfcn: int, device_index: int) -> None: device_index: SDR device index """ # Start monitoring processes - grgsm_proc, tshark_proc = _start_monitoring_processes(arfcn, device_index) + grgsm_proc, tshark_proc, field_order = _start_monitoring_processes(arfcn, device_index) app_module.gsm_spy_livemon_process = grgsm_proc app_module.gsm_spy_monitor_process = tshark_proc app_module.gsm_spy_selected_arfcn = arfcn @@ -478,7 +518,7 @@ def _start_and_register_monitor(arfcn: int, device_index: int) -> None: # Start monitoring thread monitor_thread_obj = threading.Thread( target=monitor_thread, - args=(tshark_proc,), + args=(tshark_proc, field_order), daemon=True ) monitor_thread_obj.start() @@ -1336,28 +1376,49 @@ def parse_grgsm_scanner_output(line: str) -> dict[str, Any] | None: return None -def parse_tshark_output(line: str) -> dict[str, Any] | None: - """Parse tshark filtered GSM output.""" +def parse_tshark_output(line: str, field_order: list[str] | None = None) -> dict[str, Any] | None: + """Parse tshark filtered GSM output. + + Args: + line: Tab-separated tshark output line + field_order: List of logical field names in column order. + If None, assumes legacy order: ['ta', 'tmsi', 'imsi', 'lac', 'cid'] + """ + if field_order is None: + field_order = ['ta', 'tmsi', 'imsi', 'lac', 'cid'] + try: - # tshark output format: ta_value\ttmsi\timsi\tlac\tcid parts = line.strip().split('\t') - if len(parts) >= 5: - data = { - 'type': 'device', - 'ta_value': int(parts[0]) if parts[0] else None, - 'tmsi': parts[1] if parts[1] else None, - 'imsi': parts[2] if parts[2] else None, - 'lac': int(parts[3]) if parts[3] else None, - 'cid': int(parts[4]) if parts[4] else None, - 'timestamp': datetime.now().isoformat() - } + if len(parts) < len(field_order): + return None - # Calculate distance from TA - if data['ta_value'] is not None: - data['distance_meters'] = data['ta_value'] * config.GSM_TA_METERS_PER_UNIT + # Map logical names to column values + field_map = {} + for i, logical_name in enumerate(field_order): + field_map[logical_name] = parts[i] if parts[i] else None - return data + # Convert types + ta_raw = field_map.get('ta') + data = { + 'type': 'device', + 'ta_value': int(ta_raw) if ta_raw else None, + 'tmsi': field_map.get('tmsi'), + 'imsi': field_map.get('imsi'), + 'lac': int(field_map['lac']) if field_map.get('lac') else None, + 'cid': int(field_map['cid']) if field_map.get('cid') else None, + 'timestamp': datetime.now().isoformat() + } + + # Need at least one identifier + if not data['tmsi'] and not data['imsi']: + return None + + # Calculate distance from TA + if data['ta_value'] is not None: + data['distance_meters'] = data['ta_value'] * config.GSM_TA_METERS_PER_UNIT + + return data except Exception as e: logger.debug(f"Failed to parse tshark line: {line} - {e}") @@ -1702,7 +1763,7 @@ def scanner_thread(cmd, device_index): logger.info("Monitor is running, keeping SDR device allocated") -def monitor_thread(process): +def monitor_thread(process, field_order=None): """Thread to read tshark output using standard iter pattern.""" global gsm_devices_tracked @@ -1767,7 +1828,7 @@ def monitor_thread(process): if msg_type == 'eof': break # EOF - parsed = parse_tshark_output(line) + parsed = parse_tshark_output(line, field_order) if parsed: packets_captured += 1 From 3dc16b392b7cabeedf4623c38ad4ab5a86820252 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 19:24:10 +0000 Subject: [PATCH 06/24] Remove tshark -Y display filter that blocked all GSM packets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The display filter `gsm_a.tmsi || e212.imsi` was too restrictive — paging requests use different field paths for TMSI so nothing matched. The capture filter (-f 'udp port 4729') already limits to GSMTAP, and the parser discards rows without TMSI/IMSI identifiers. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index a171e16..e0e9563 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -447,14 +447,10 @@ def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subproce '-f', 'udp port 4729', # Capture filter: only GSMTAP packets ] - # Build display filter from available fields - filter_parts = [] - for logical_name in ['ta', 'tmsi', 'imsi']: - if fields.get(logical_name): - filter_parts.append(fields[logical_name]) - if filter_parts: - tshark_cmd.extend(['-Y', ' || '.join(filter_parts)]) - + # No display filter (-Y) — the capture filter (-f 'udp port 4729') + # already limits to GSMTAP packets, and the parser discards rows + # without TMSI/IMSI. A -Y filter on gsm_a.tmsi misses paging + # requests where the TMSI lives under a different field path. tshark_cmd.extend(['-T', 'fields']) # Add -e for each available field in known order From 391aff52ce07d75754cce18bf04d78700e64b08a Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 19:37:06 +0000 Subject: [PATCH 07/24] Fix OpenCellID integration: CID=0 handling, API key check, tab parsing - /lookup_cell and /detect_rogue rejected CID=0 towers because `all([..., cid])` is falsy when cid=0; use `is not None` checks - can_use_api() now returns False when GSM_OPENCELLID_API_KEY is empty, preventing the geocoding worker from wasting daily quota on doomed calls - /lookup_cell returns 503 with clear message when API key not configured - parse_tshark_output uses rstrip('\n\r') instead of strip() to preserve leading empty tab-separated fields (strip() ate leading tabs, shifting all columns when the first field was empty) Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 12 ++++++++---- tests/test_gsm_spy.py | 31 ++++++++++++++++++++++--------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index e0e9563..adf080f 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -89,6 +89,8 @@ def increment_api_usage(): def can_use_api(): """Check if we can make an API call within daily limit.""" + if not config.GSM_OPENCELLID_API_KEY: + return False current_usage = get_api_usage_today() return current_usage < config.GSM_API_DAILY_LIMIT @@ -800,7 +802,7 @@ def lookup_cell(): lac = data.get('lac') cid = data.get('cid') - if not all([mcc, mnc, lac, cid]): + if any(v is None for v in [mcc, mnc, lac, cid]): return jsonify({'error': 'MCC, MNC, LAC, and CID required'}), 400 try: @@ -823,7 +825,9 @@ def lookup_cell(): 'radio': result['radio'] }) - # Check API usage limit + # Check API key and usage limit + if not config.GSM_OPENCELLID_API_KEY: + return jsonify({'error': 'OpenCellID API key not configured'}), 503 if not can_use_api(): current_usage = get_api_usage_today() return jsonify({ @@ -905,7 +909,7 @@ def detect_rogue(): lac = tower_info.get('lac') cid = tower_info.get('cid') - if all([mcc, mnc, lac, cid]): + if all(v is not None for v in [mcc, mnc, lac, cid]): with get_db() as conn: result = conn.execute(''' SELECT id FROM gsm_cells @@ -1384,7 +1388,7 @@ def parse_tshark_output(line: str, field_order: list[str] | None = None) -> dict field_order = ['ta', 'tmsi', 'imsi', 'lac', 'cid'] try: - parts = line.strip().split('\t') + parts = line.rstrip('\n\r').split('\t') if len(parts) < len(field_order): return None diff --git a/tests/test_gsm_spy.py b/tests/test_gsm_spy.py index fbcf6bb..4ab8815 100644 --- a/tests/test_gsm_spy.py +++ b/tests/test_gsm_spy.py @@ -120,25 +120,38 @@ class TestParseTsharkOutput: assert 'timestamp' in result def test_missing_optional_fields(self): - """Test parsing with missing optional fields (empty tabs).""" + """Test parsing with missing optional fields (empty tabs). + + A packet with TA but no TMSI/IMSI is discarded since there's + no device identifier to track. + """ line = "3\t\t\t1234\t31245" result = parse_tshark_output(line) + assert result is None + + def test_missing_optional_fields_with_tmsi(self): + """Test parsing with TMSI but missing TA, IMSI, CID.""" + line = "\t0xABCD\t\t1234\t" + result = parse_tshark_output(line) assert result is not None - assert result['ta_value'] == 3 - assert result['tmsi'] is None + assert result['ta_value'] is None + assert result['tmsi'] == '0xABCD' assert result['imsi'] is None assert result['lac'] == 1234 - assert result['cid'] == 31245 + assert result['cid'] is None def test_no_ta_value(self): - """Test parsing without TA value (empty field).""" - # When TA is empty, int('') will fail, so the parse returns None - # This is the current behavior - the function expects valid integers or valid empty handling + """Test parsing without TA value (empty first field).""" line = "\t0xABCD1234\t123456789012345\t1234\t31245" result = parse_tshark_output(line) - # Current implementation will fail to parse this due to int('') failing - assert result is None + + assert result is not None + assert result['ta_value'] is None + assert result['tmsi'] == '0xABCD1234' + assert result['imsi'] == '123456789012345' + assert result['lac'] == 1234 + assert result['cid'] == 31245 def test_invalid_line(self): """Test handling of invalid tshark output.""" From c6a8a4a492c64d87e52da871836129a80ab6816c Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 19:50:04 +0000 Subject: [PATCH 08/24] Fix EGSM900 downlink frequency: 935 MHz not 925 MHz MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EGSM900 band table had start=925e6 but ARFCNs 0-124 use downlink frequencies starting at 935 MHz (DL = 935 + 0.2*ARFCN). The 925 MHz value is the E-GSM extension band (ARFCNs 975-1023). This caused grgsm_livemon to tune 10 MHz too low — ARFCN 22 tuned to 929.4 MHz instead of 939.4 MHz, receiving no GSM frames and producing zero GSMTAP packets for tshark to capture. Also adds EGSM900_EXT band (ARFCNs 975-1023, DL 925.2-934.8 MHz) and diagnostic logging in the monitor thread to track raw tshark line counts vs parsed packets. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 18 ++++++++++++++++-- tests/test_gsm_spy.py | 21 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index adf080f..9086247 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -40,11 +40,13 @@ REGIONAL_BANDS = { 'Europe': { 'GSM800': {'start': 832e6, 'end': 862e6, 'arfcn_start': 438, 'arfcn_end': 511}, # E-GSM800 downlink 'GSM850': {'start': 869e6, 'end': 894e6, 'arfcn_start': 128, 'arfcn_end': 251}, # Also used in some EU countries - 'EGSM900': {'start': 925e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, + 'EGSM900': {'start': 935e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, # DL = 935 + 0.2*ARFCN + 'EGSM900_EXT': {'start': 925.2e6, 'end': 935e6, 'arfcn_start': 975, 'arfcn_end': 1023}, # E-GSM extension 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} }, 'Asia': { - 'EGSM900': {'start': 925e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, + 'EGSM900': {'start': 935e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, # DL = 935 + 0.2*ARFCN + 'EGSM900_EXT': {'start': 925.2e6, 'end': 935e6, 'arfcn_start': 975, 'arfcn_end': 1023}, # E-GSM extension 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} } } @@ -1795,6 +1797,7 @@ def monitor_thread(process, field_order=None): monitor_start_time = time.time() packets_captured = 0 + lines_received = 0 last_heartbeat = time.time() try: @@ -1818,6 +1821,12 @@ def monitor_thread(process, field_order=None): }) except queue.Full: pass + # Periodic diagnostic: how many raw lines vs parsed + if lines_received > 0 or elapsed % 30 == 0: + logger.info( + f"Monitor stats: {lines_received} tshark lines received, " + f"{packets_captured} parsed, fields={field_order}" + ) # Get output from queue with timeout try: @@ -1828,6 +1837,11 @@ def monitor_thread(process, field_order=None): if msg_type == 'eof': break # EOF + lines_received += 1 + # Log first 5 raw lines and then every 100th for diagnostics + if lines_received <= 5 or lines_received % 100 == 0: + logger.debug(f"tshark raw line #{lines_received}: {line.rstrip()!r}") + parsed = parse_tshark_output(line, field_order) if parsed: packets_captured += 1 diff --git a/tests/test_gsm_spy.py b/tests/test_gsm_spy.py index 4ab8815..797f794 100644 --- a/tests/test_gsm_spy.py +++ b/tests/test_gsm_spy.py @@ -187,14 +187,29 @@ class TestArfcnToFrequency: def test_egsm900_arfcn(self): """Test ARFCN in EGSM900 band.""" - # EGSM900: ARFCN 0-124, 925-960 MHz + # EGSM900: ARFCN 0-124, DL = 935 + 0.2*ARFCN MHz arfcn = 0 freq = arfcn_to_frequency(arfcn) - assert freq == 925000000 # 925 MHz + assert freq == 935000000 # 935.0 MHz + + arfcn = 22 + freq = arfcn_to_frequency(arfcn) + assert freq == 939400000 # 939.4 MHz arfcn = 124 freq = arfcn_to_frequency(arfcn) - assert freq == 949800000 # 949.8 MHz + assert freq == 959800000 # 959.8 MHz + + def test_egsm900_ext_arfcn(self): + """Test ARFCN in EGSM900 extension band.""" + # EGSM900_EXT: ARFCN 975-1023, DL = 925.2 + 0.2*(ARFCN-975) MHz + arfcn = 975 + freq = arfcn_to_frequency(arfcn) + assert freq == 925200000 # 925.2 MHz + + arfcn = 1023 + freq = arfcn_to_frequency(arfcn) + assert freq == 934800000 # 934.8 MHz def test_dcs1800_arfcn(self): """Test ARFCN in DCS1800 band.""" From 7d69cac7e7879ff5cef5a47038a97666da367035 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 19:55:00 +0000 Subject: [PATCH 09/24] Fix geocoding: validate API responses, clean poisoned cache, improve logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Cache lookup now requires non-NULL lat/lon — previously a row with NULL coordinates counted as a cache hit, returning {lat: None, lon: None} which the frontend silently ignored (tower in list but no map pin) - API response handler validates lat/lon exist before caching, preventing error responses (status 200 with error body) from poisoning the cache - On geocoding worker start, delete any existing poisoned cache rows - Geocoding worker now logs "API key not configured" vs "rate limit reached" so the actual problem is visible in logs - API error responses now log the response body for easier debugging --- routes/gsm_spy.py | 22 +++++++++++++++++++--- utils/gsm_geocoding.py | 26 +++++++++++++++++++------- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index 9086247..90b419f 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -104,6 +104,19 @@ def can_use_api(): def start_geocoding_worker(): """Start background thread for async geocoding.""" global _geocoding_worker_thread + + # Clean poisoned cache entries (rows with NULL lat/lon from failed API responses) + try: + with get_db() as conn: + deleted = conn.execute( + 'DELETE FROM gsm_cells WHERE lat IS NULL OR lon IS NULL' + ).rowcount + conn.commit() + if deleted: + logger.info(f"Cleaned {deleted} poisoned cache entries (NULL coordinates)") + except Exception as e: + logger.warning(f"Could not clean cache: {e}") + if _geocoding_worker_thread is None or not _geocoding_worker_thread.is_alive(): _geocoding_worker_thread = threading.Thread( target=geocoding_worker, @@ -125,10 +138,13 @@ def geocoding_worker(): # Wait for pending tower with timeout tower_data = geocoding_queue.get(timeout=5) - # Check rate limit + # Check API key and rate limit if not can_use_api(): - current_usage = get_api_usage_today() - logger.warning(f"OpenCellID API rate limit reached ({current_usage}/{config.GSM_API_DAILY_LIMIT})") + if not config.GSM_OPENCELLID_API_KEY: + logger.warning("OpenCellID API key not configured (set INTERCEPT_GSM_OPENCELLID_API_KEY)") + else: + current_usage = get_api_usage_today() + logger.warning(f"OpenCellID API daily limit reached ({current_usage}/{config.GSM_API_DAILY_LIMIT})") geocoding_queue.task_done() continue diff --git a/utils/gsm_geocoding.py b/utils/gsm_geocoding.py index 681b990..006368d 100644 --- a/utils/gsm_geocoding.py +++ b/utils/gsm_geocoding.py @@ -47,7 +47,7 @@ def lookup_cell_coordinates(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? ''', (mcc, mnc, lac, cid)).fetchone() - if result: + if result and result['lat'] is not None and result['lon'] is not None: return { 'lat': result['lat'], 'lon': result['lon'], @@ -95,6 +95,16 @@ def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, An if response.status_code == 200: cell_data = response.json() + lat = cell_data.get('lat') + lon = cell_data.get('lon') + + # Validate response has actual coordinates + if lat is None or lon is None: + logger.warning( + f"OpenCellID API returned 200 but no coordinates for " + f"MCC={mcc} MNC={mnc} LAC={lac} CID={cid}: {cell_data}" + ) + return None # Cache the result with get_db() as conn: @@ -104,8 +114,7 @@ def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, An VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) ''', ( mcc, mnc, lac, cid, - cell_data.get('lat'), - cell_data.get('lon'), + lat, lon, cell_data.get('azimuth'), cell_data.get('range'), cell_data.get('samples'), @@ -114,11 +123,11 @@ def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, An )) conn.commit() - logger.info(f"Cached cell tower from API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") + logger.info(f"Cached cell tower from API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid} -> ({lat}, {lon})") return { - 'lat': cell_data.get('lat'), - 'lon': cell_data.get('lon'), + 'lat': lat, + 'lon': lon, 'source': 'api', 'azimuth': cell_data.get('azimuth'), 'range_meters': cell_data.get('range'), @@ -126,7 +135,10 @@ def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, An 'radio': cell_data.get('radio') } else: - logger.warning(f"OpenCellID API returned {response.status_code} for MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") + logger.warning( + f"OpenCellID API returned {response.status_code} for " + f"MCC={mcc} MNC={mnc} LAC={lac} CID={cid}: {response.text[:200]}" + ) return None except Exception as e: From 98f6d18bea54e567828a96cbe8095c1b046c1990 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 20:24:51 +0000 Subject: [PATCH 10/24] Fix GSM dashboard counters, improve lists, add device detail modal Wire SIGNALS/DEVICES/CROWD counters to monitor_heartbeat SSE data so they update in real-time during monitoring. Redesign device list items as richer cards with type badges, TA/distance, and observation counts. Add clickable device detail modal with full device info and copy support. Improve tower list with signal strength bars. Widen right sidebar and bump list font sizes for readability. Co-Authored-By: Claude Opus 4.6 --- templates/gsm_spy_dashboard.html | 507 +++++++++++++++++++++++++++++-- 1 file changed, 483 insertions(+), 24 deletions(-) diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html index ff05a7c..3cdc173 100644 --- a/templates/gsm_spy_dashboard.html +++ b/templates/gsm_spy_dashboard.html @@ -428,7 +428,7 @@ position: relative; z-index: 10; display: grid; - grid-template-columns: 280px 1fr 300px; + grid-template-columns: 280px 1fr 340px; grid-template-rows: 1fr auto; gap: 0; height: calc(100vh - 160px); @@ -821,16 +821,15 @@ } .tracked-list-content { - max-height: 400px; overflow-y: auto; } .list-item { - padding: 10px 12px; + padding: 12px 14px; border-bottom: 1px solid var(--border-color); cursor: pointer; transition: background 0.2s; - font-size: 11px; + font-size: 12px; } .list-item:hover { @@ -861,9 +860,9 @@ } .list-item-details { - font-size: 10px; + font-size: 11px; color: var(--text-secondary); - line-height: 1.4; + line-height: 1.5; } .rogue-indicator { @@ -1098,10 +1097,289 @@ /* Responsive adjustments */ @media (max-width: 1400px) { .dashboard { - grid-template-columns: 250px 1fr 280px; + grid-template-columns: 250px 1fr 300px; } } + /* Signal Strength Bar */ + .signal-bar-container { + display: flex; + align-items: center; + gap: 6px; + margin-top: 4px; + } + + .signal-bar-track { + flex: 1; + height: 4px; + background: rgba(255, 255, 255, 0.08); + border-radius: 2px; + overflow: hidden; + } + + .signal-bar-fill { + height: 100%; + border-radius: 2px; + transition: width 0.3s ease; + } + + .signal-bar-label { + font-size: 10px; + font-family: var(--font-mono); + color: var(--text-dim); + min-width: 50px; + text-align: right; + } + + /* Device Card Styles */ + .device-card { + padding: 12px 14px; + border-bottom: 1px solid var(--border-color); + cursor: pointer; + transition: all 0.2s; + font-size: 12px; + } + + .device-card:hover { + background: rgba(74, 163, 255, 0.08); + border-left: 3px solid var(--accent-cyan); + } + + .device-card-top { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 6px; + } + + .device-card-id { + font-weight: 700; + font-family: var(--font-mono); + font-size: 12px; + display: flex; + align-items: center; + gap: 6px; + } + + .device-type-badge { + display: inline-block; + padding: 1px 6px; + border-radius: 3px; + font-size: 9px; + font-weight: 700; + letter-spacing: 0.5px; + text-transform: uppercase; + } + + .device-type-badge.imsi { + background: rgba(56, 193, 128, 0.2); + color: var(--accent-green); + border: 1px solid rgba(56, 193, 128, 0.3); + } + + .device-type-badge.tmsi { + background: rgba(74, 163, 255, 0.2); + color: var(--accent-cyan); + border: 1px solid rgba(74, 163, 255, 0.3); + } + + .device-card-time { + font-size: 10px; + color: var(--text-dim); + font-family: var(--font-mono); + } + + .device-card-mid { + display: flex; + gap: 12px; + font-size: 11px; + color: var(--text-secondary); + margin-bottom: 4px; + } + + .device-card-mid span { + display: flex; + align-items: center; + gap: 3px; + } + + .device-card-bottom { + display: flex; + justify-content: space-between; + align-items: center; + font-size: 10px; + } + + .device-seen-badge { + padding: 1px 6px; + border-radius: 3px; + font-size: 9px; + font-weight: 600; + } + + .device-seen-badge.new { + background: rgba(225, 194, 107, 0.15); + color: var(--accent-yellow); + } + + .device-seen-badge.returning { + background: rgba(56, 193, 128, 0.15); + color: var(--accent-green); + } + + /* Device Detail Modal */ + .device-detail-modal { + display: none; + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + z-index: 2000; + animation: fadeIn 0.2s ease-out; + } + + .device-detail-modal.active { + display: flex; + justify-content: center; + align-items: center; + } + + .device-detail-backdrop { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(11, 17, 24, 0.9); + backdrop-filter: blur(4px); + } + + .device-detail-content { + position: relative; + background: var(--bg-panel); + border: 1px solid var(--border-color); + border-radius: 8px; + width: 420px; + max-width: 90vw; + max-height: 80vh; + overflow: hidden; + display: flex; + flex-direction: column; + box-shadow: 0 0 40px rgba(74, 163, 255, 0.25); + animation: slideUp 0.3s ease-out; + } + + .device-detail-header { + padding: 14px 18px; + background: var(--bg-card); + border-bottom: 1px solid var(--border-color); + display: flex; + justify-content: space-between; + align-items: center; + } + + .device-detail-title { + font-size: 14px; + font-weight: 700; + letter-spacing: 1px; + color: var(--accent-cyan); + text-transform: uppercase; + } + + .device-detail-close { + width: 28px; + height: 28px; + border: 1px solid var(--border-color); + background: var(--bg-dark); + color: var(--text-secondary); + border-radius: 4px; + cursor: pointer; + transition: all 0.2s; + font-size: 16px; + display: flex; + align-items: center; + justify-content: center; + } + + .device-detail-close:hover { + border-color: var(--accent-red); + color: var(--accent-red); + } + + .device-detail-body { + padding: 16px 18px; + overflow-y: auto; + flex: 1; + } + + .detail-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 12px; + margin-bottom: 16px; + } + + .detail-field { + display: flex; + flex-direction: column; + gap: 3px; + } + + .detail-field.full-width { + grid-column: 1 / -1; + } + + .detail-field-label { + font-size: 9px; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; + font-family: var(--font-mono); + } + + .detail-field-value { + font-size: 13px; + font-weight: 600; + color: var(--text-primary); + font-family: var(--font-mono); + display: flex; + align-items: center; + gap: 6px; + } + + .detail-copy-btn { + background: none; + border: 1px solid var(--border-color); + color: var(--text-dim); + border-radius: 3px; + padding: 2px 6px; + font-size: 9px; + cursor: pointer; + transition: all 0.2s; + font-family: var(--font-mono); + } + + .detail-copy-btn:hover { + border-color: var(--accent-cyan); + color: var(--accent-cyan); + } + + .detail-section { + border-top: 1px solid var(--border-color); + padding-top: 12px; + margin-top: 4px; + } + + .detail-section-title { + font-size: 10px; + font-weight: 700; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 1px; + margin-bottom: 10px; + } + @media (max-width: 1024px) { .dashboard { grid-template-columns: 1fr; @@ -1133,6 +1411,16 @@ {% set active_mode = 'gsm' %} {% include 'partials/nav.html' with context %} + + + + {% include 'partials/settings-modal.html' %} +
@@ -1488,6 +1776,19 @@
+ +
+
+
+
+ Device Detail + +
+
+
+
+
+ + + From 90e88fc4695d235939faf7c2f53257d33305ffe9 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 20:35:31 +0000 Subject: [PATCH 11/24] Fix tshark hex parsing and add API key settings UI Parse tshark GSM field values with int(value, 0) instead of int(value) to auto-detect hex 0x-prefixed output (e.g. 0x039e for TMSI/LAC/CID). Without this, every tshark line with hex values fails to parse, causing 0 devices to be captured during monitoring. Also add API Keys tab to Settings modal for configuring OpenCellID key via the UI (in addition to env var), with status display and usage bar. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 76 +++++++++++++++++--- static/js/core/settings-manager.js | 96 ++++++++++++++++++++++++++ templates/partials/settings-modal.html | 65 +++++++++++++++++ utils/gsm_geocoding.py | 16 ++++- 4 files changed, 242 insertions(+), 11 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index 90b419f..fee519e 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -89,9 +89,18 @@ def increment_api_usage(): return current + 1 +def get_opencellid_api_key(): + """Get OpenCellID API key, checking env var first, then database setting.""" + env_key = config.GSM_OPENCELLID_API_KEY + if env_key: + return env_key + from utils.database import get_setting + return get_setting('gsm.opencellid.api_key', '') + + def can_use_api(): """Check if we can make an API call within daily limit.""" - if not config.GSM_OPENCELLID_API_KEY: + if not get_opencellid_api_key(): return False current_usage = get_api_usage_today() return current_usage < config.GSM_API_DAILY_LIMIT @@ -140,8 +149,8 @@ def geocoding_worker(): # Check API key and rate limit if not can_use_api(): - if not config.GSM_OPENCELLID_API_KEY: - logger.warning("OpenCellID API key not configured (set INTERCEPT_GSM_OPENCELLID_API_KEY)") + if not get_opencellid_api_key(): + logger.warning("OpenCellID API key not configured (set INTERCEPT_GSM_OPENCELLID_API_KEY or configure in Settings > API Keys)") else: current_usage = get_api_usage_today() logger.warning(f"OpenCellID API daily limit reached ({current_usage}/{config.GSM_API_DAILY_LIMIT})") @@ -807,7 +816,53 @@ def status(): 'selected_arfcn': app_module.gsm_spy_selected_arfcn, 'api_usage_today': api_usage, 'api_limit': config.GSM_API_DAILY_LIMIT, - 'api_remaining': config.GSM_API_DAILY_LIMIT - api_usage + 'api_remaining': config.GSM_API_DAILY_LIMIT - api_usage, + 'api_key_configured': bool(get_opencellid_api_key()) + }) + + +@gsm_spy_bp.route('/settings/api_key', methods=['GET', 'POST']) +def settings_api_key(): + """Get or set OpenCellID API key configuration.""" + from utils.database import get_setting, set_setting + + if request.method == 'GET': + env_key = config.GSM_OPENCELLID_API_KEY + db_key = get_setting('gsm.opencellid.api_key', '') + + if env_key: + source = 'env' + configured = True + elif db_key: + source = 'database' + configured = True + else: + source = 'none' + configured = False + + usage_today = get_api_usage_today() + + return jsonify({ + 'configured': configured, + 'source': source, + 'usage_today': usage_today, + 'api_limit': config.GSM_API_DAILY_LIMIT + }) + + # POST: save key to database + data = request.get_json() or {} + key = data.get('key', '').strip() + + if not key: + return jsonify({'error': 'API key cannot be empty'}), 400 + + set_setting('gsm.opencellid.api_key', key) + logger.info("OpenCellID API key saved to database") + + return jsonify({ + 'status': 'saved', + 'configured': True, + 'source': 'database' }) @@ -844,7 +899,8 @@ def lookup_cell(): }) # Check API key and usage limit - if not config.GSM_OPENCELLID_API_KEY: + api_key = get_opencellid_api_key() + if not api_key: return jsonify({'error': 'OpenCellID API key not configured'}), 503 if not can_use_api(): current_usage = get_api_usage_today() @@ -857,7 +913,7 @@ def lookup_cell(): # Call OpenCellID API api_url = config.GSM_OPENCELLID_API_URL params = { - 'key': config.GSM_OPENCELLID_API_KEY, + 'key': api_key, 'mcc': mcc, 'mnc': mnc, 'lac': lac, @@ -1416,15 +1472,15 @@ def parse_tshark_output(line: str, field_order: list[str] | None = None) -> dict for i, logical_name in enumerate(field_order): field_map[logical_name] = parts[i] if parts[i] else None - # Convert types + # Convert types (use base 0 to auto-detect hex 0x prefix from tshark) ta_raw = field_map.get('ta') data = { 'type': 'device', - 'ta_value': int(ta_raw) if ta_raw else None, + 'ta_value': int(ta_raw, 0) if ta_raw else None, 'tmsi': field_map.get('tmsi'), 'imsi': field_map.get('imsi'), - 'lac': int(field_map['lac']) if field_map.get('lac') else None, - 'cid': int(field_map['cid']) if field_map.get('cid') else None, + 'lac': int(field_map['lac'], 0) if field_map.get('lac') else None, + 'cid': int(field_map['cid'], 0) if field_map.get('cid') else None, 'timestamp': datetime.now().isoformat() } diff --git a/static/js/core/settings-manager.js b/static/js/core/settings-manager.js index 3d2cb8b..69ed89b 100644 --- a/static/js/core/settings-manager.js +++ b/static/js/core/settings-manager.js @@ -930,5 +930,101 @@ function switchSettingsTab(tabName) { if (typeof RecordingUI !== 'undefined') { RecordingUI.refresh(); } + } else if (tabName === 'apikeys') { + loadApiKeyStatus(); } } + +/** + * Load API key status into the API Keys settings tab + */ +function loadApiKeyStatus() { + const badge = document.getElementById('apiKeyStatusBadge'); + const desc = document.getElementById('apiKeyStatusDesc'); + const usage = document.getElementById('apiKeyUsageCount'); + const bar = document.getElementById('apiKeyUsageBar'); + + if (!badge) return; + + fetch('/gsm_spy/settings/api_key') + .then(r => r.json()) + .then(data => { + if (data.configured) { + badge.textContent = 'Configured'; + badge.className = 'asset-badge available'; + desc.textContent = 'Source: ' + (data.source === 'env' ? 'Environment variable' : 'Database'); + } else { + badge.textContent = 'Not configured'; + badge.className = 'asset-badge missing'; + desc.textContent = 'No API key set'; + } + if (usage) { + usage.textContent = (data.usage_today || 0) + ' / ' + (data.api_limit || 1000); + } + if (bar) { + const pct = Math.min(100, ((data.usage_today || 0) / (data.api_limit || 1000)) * 100); + bar.style.width = pct + '%'; + bar.style.background = pct > 90 ? 'var(--accent-red)' : pct > 70 ? 'var(--accent-yellow)' : 'var(--accent-cyan)'; + } + }) + .catch(() => { + badge.textContent = 'Error'; + badge.className = 'asset-badge missing'; + desc.textContent = 'Could not load status'; + }); +} + +/** + * Save API key from the settings input + */ +function saveApiKey() { + const input = document.getElementById('apiKeyInput'); + const result = document.getElementById('apiKeySaveResult'); + if (!input || !result) return; + + const key = input.value.trim(); + if (!key) { + result.style.display = 'block'; + result.style.color = 'var(--accent-red)'; + result.textContent = 'Please enter an API key.'; + return; + } + + result.style.display = 'block'; + result.style.color = 'var(--text-dim)'; + result.textContent = 'Saving...'; + + fetch('/gsm_spy/settings/api_key', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ key: key }) + }) + .then(r => r.json()) + .then(data => { + if (data.error) { + result.style.color = 'var(--accent-red)'; + result.textContent = data.error; + } else { + result.style.color = 'var(--accent-green)'; + result.textContent = 'API key saved successfully.'; + input.value = ''; + loadApiKeyStatus(); + // Hide the banner if visible + const banner = document.getElementById('apiKeyBanner'); + if (banner) banner.style.display = 'none'; + } + }) + .catch(() => { + result.style.color = 'var(--accent-red)'; + result.textContent = 'Error saving API key.'; + }); +} + +/** + * Toggle API key input visibility + */ +function toggleApiKeyVisibility() { + const input = document.getElementById('apiKeyInput'); + if (!input) return; + input.type = input.type === 'password' ? 'text' : 'password'; +} diff --git a/templates/partials/settings-modal.html b/templates/partials/settings-modal.html index b9c951b..ca890ab 100644 --- a/templates/partials/settings-modal.html +++ b/templates/partials/settings-modal.html @@ -17,6 +17,7 @@ +
@@ -359,6 +360,70 @@ + +
+
+
OpenCellID API Key
+

+ Required for GSM cell tower geolocation. Get a free key at + opencellid.org/register + (1,000 lookups/day). +

+ +
+
+ Status + Checking... +
+ Checking... +
+ +
+
+ API Key + Paste your OpenCellID API token +
+
+ + +
+
+ +
+ +
+ + +
+ +
+
Usage Today
+
+
+ API Calls + -- / -- +
+
+
+
+
+
+ +
+ Note: The environment variable INTERCEPT_GSM_OPENCELLID_API_KEY takes priority over the saved key. + Keys saved here persist across restarts. +
+
+
diff --git a/utils/gsm_geocoding.py b/utils/gsm_geocoding.py index 006368d..feaf164 100644 --- a/utils/gsm_geocoding.py +++ b/utils/gsm_geocoding.py @@ -65,6 +65,15 @@ def lookup_cell_coordinates(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, return None +def _get_api_key() -> str: + """Get OpenCellID API key at runtime (env var first, then database).""" + env_key = config.GSM_OPENCELLID_API_KEY + if env_key: + return env_key + from utils.database import get_setting + return get_setting('gsm.opencellid.api_key', '') + + def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None: """ Lookup cell tower from OpenCellID API and cache result. @@ -81,9 +90,14 @@ def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, An Returns None if API call fails or cell not found. """ try: + api_key = _get_api_key() + if not api_key: + logger.warning("OpenCellID API key not configured") + return None + api_url = config.GSM_OPENCELLID_API_URL params = { - 'key': config.GSM_OPENCELLID_API_KEY, + 'key': api_key, 'mcc': mcc, 'mnc': mnc, 'lac': lac, From 1a5b076a8dea68159756f1fbca50a1d8371b5463 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 20:39:12 +0000 Subject: [PATCH 12/24] Fix grgsm_scanner crash on unsupported band names (GSM800, EGSM900_EXT) Add explicit band name mapping from internal names to grgsm_scanner's accepted -b values (GSM900, GSM850, DCS1800, PCS1900). Bands without a valid grgsm_scanner equivalent (GSM800, EGSM900_EXT) are skipped with a log message instead of crashing the scanner. Remove GSM800 from the dashboard band selector since it can't be scanned. Co-Authored-By: Claude Opus 4.6 --- routes/gsm_spy.py | 28 ++++++++++++++++++++++++---- templates/gsm_spy_dashboard.html | 3 +-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py index fee519e..49aca53 100644 --- a/routes/gsm_spy.py +++ b/routes/gsm_spy.py @@ -614,11 +614,31 @@ def start_scanner(): cmd.extend(['--args', f'rtl={device_index}']) # Add selected band arguments - # Map EGSM900 to GSM900 since that's what grgsm_scanner expects + # Map internal band names to grgsm_scanner -b values + # grgsm_scanner accepts: GSM900, GSM850, DCS1800, PCS1900, GSM450, GSM480, GSM-R + GRGSM_BAND_MAP = { + 'EGSM900': 'GSM900', + 'EGSM900_EXT': None, # Covered by GSM900 scan + 'GSM850': 'GSM850', + 'GSM800': None, # Not a standard GSM band for grgsm_scanner + 'DCS1800': 'DCS1800', + 'PCS1900': 'PCS1900', + } + bands_added = set() for band_name in selected_bands: - # Normalize band name (EGSM900 -> GSM900, remove EGSM prefix) - normalized_band = band_name.replace('EGSM', 'GSM') - cmd.extend(['-b', normalized_band]) + grgsm_band = GRGSM_BAND_MAP.get(band_name, band_name) + if grgsm_band is None: + logger.info(f"Skipping band {band_name} (not supported by grgsm_scanner)") + continue + if grgsm_band not in bands_added: + cmd.extend(['-b', grgsm_band]) + bands_added.add(grgsm_band) + + if not bands_added: + from app import release_sdr_device + release_sdr_device(device_index) + return jsonify({'error': f'No scannable bands selected. ' + f'GSM800 and EGSM900_EXT are not supported by grgsm_scanner.'}), 400 logger.info(f"Starting GSM scanner: {' '.join(cmd)}") diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html index 3cdc173..3cb953f 100644 --- a/templates/gsm_spy_dashboard.html +++ b/templates/gsm_spy_dashboard.html @@ -1826,8 +1826,7 @@ 'Europe': [ { name: 'EGSM900', label: 'EGSM900 (925-960 MHz)', freq: '925-960 MHz', common: true, recommended: true }, { name: 'DCS1800', label: 'DCS1800 (1805-1880 MHz)', freq: '1805-1880 MHz', common: true, recommended: false }, - { name: 'GSM850', label: 'GSM850 (869-894 MHz)', freq: '869-894 MHz', common: false, recommended: false }, - { name: 'GSM800', label: 'GSM800 (832-862 MHz)', freq: '832-862 MHz', common: false, recommended: false } + { name: 'GSM850', label: 'GSM850 (869-894 MHz)', freq: '869-894 MHz', common: false, recommended: false } ], 'Americas': [ { name: 'GSM850', label: 'GSM850 (869-894 MHz)', freq: '869-894 MHz', common: true, recommended: true }, From eff6ca3e87827d5d7b3816c6dafe21fc11bffe50 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 20:41:02 +0000 Subject: [PATCH 13/24] Add 2G generation label to GSM band selector options Co-Authored-By: Claude Opus 4.6 --- templates/gsm_spy_dashboard.html | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html index 3cb953f..cfadbb3 100644 --- a/templates/gsm_spy_dashboard.html +++ b/templates/gsm_spy_dashboard.html @@ -1824,17 +1824,17 @@ // Band configurations by region const BAND_CONFIG = { 'Europe': [ - { name: 'EGSM900', label: 'EGSM900 (925-960 MHz)', freq: '925-960 MHz', common: true, recommended: true }, - { name: 'DCS1800', label: 'DCS1800 (1805-1880 MHz)', freq: '1805-1880 MHz', common: true, recommended: false }, - { name: 'GSM850', label: 'GSM850 (869-894 MHz)', freq: '869-894 MHz', common: false, recommended: false } + { name: 'EGSM900', label: '2G · EGSM900 (925-960 MHz)', freq: '925-960 MHz', common: true, recommended: true }, + { name: 'DCS1800', label: '2G · DCS1800 (1805-1880 MHz)', freq: '1805-1880 MHz', common: true, recommended: false }, + { name: 'GSM850', label: '2G · GSM850 (869-894 MHz)', freq: '869-894 MHz', common: false, recommended: false } ], 'Americas': [ - { name: 'GSM850', label: 'GSM850 (869-894 MHz)', freq: '869-894 MHz', common: true, recommended: true }, - { name: 'PCS1900', label: 'PCS1900 (1930-1990 MHz)', freq: '1930-1990 MHz', common: true, recommended: true } + { name: 'GSM850', label: '2G · GSM850 (869-894 MHz)', freq: '869-894 MHz', common: true, recommended: true }, + { name: 'PCS1900', label: '2G · PCS1900 (1930-1990 MHz)', freq: '1930-1990 MHz', common: true, recommended: true } ], 'Asia': [ - { name: 'EGSM900', label: 'EGSM900 (925-960 MHz)', freq: '925-960 MHz', common: true, recommended: true }, - { name: 'DCS1800', label: 'DCS1800 (1805-1880 MHz)', freq: '1805-1880 MHz', common: true, recommended: true } + { name: 'EGSM900', label: '2G · EGSM900 (925-960 MHz)', freq: '925-960 MHz', common: true, recommended: true }, + { name: 'DCS1800', label: '2G · DCS1800 (1805-1880 MHz)', freq: '1805-1880 MHz', common: true, recommended: true } ] }; From c6ff8abf113b568f1eac57ea88b4a232075cba91 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 20:55:06 +0000 Subject: [PATCH 14/24] Add Leaflet.heat crowd density heatmap to GSM Spy dashboard Adds a toggleable heatmap layer that visualizes crowd density data from the existing /gsm_spy/crowd_density endpoint as a gradient overlay on the map, with auto-refresh every 30s during active monitoring. Co-Authored-By: Claude Opus 4.6 --- routes/offline.py | 5 +- setup.sh | 13 +++ static/vendor/leaflet-heat/leaflet-heat.js | 11 +++ templates/gsm_spy_dashboard.html | 92 ++++++++++++++++++++++ 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 static/vendor/leaflet-heat/leaflet-heat.js diff --git a/routes/offline.py b/routes/offline.py index a35b6e9..ff25922 100644 --- a/routes/offline.py +++ b/routes/offline.py @@ -13,7 +13,7 @@ OFFLINE_DEFAULTS = { 'offline.enabled': False, 'offline.assets_source': 'cdn', 'offline.fonts_source': 'cdn', - 'offline.tile_provider': 'cartodb_dark_cyan', + 'offline.tile_provider': 'cartodb_dark_cyan', 'offline.tile_server_url': '' } @@ -44,6 +44,9 @@ ASSET_PATHS = { 'static/vendor/leaflet/images/marker-shadow.png', 'static/vendor/leaflet/images/layers.png', 'static/vendor/leaflet/images/layers-2x.png' + ], + 'leaflet_heat': [ + 'static/vendor/leaflet-heat/leaflet-heat.js' ] } diff --git a/setup.sh b/setup.sh index f41218a..3e0f210 100755 --- a/setup.sh +++ b/setup.sh @@ -1375,6 +1375,19 @@ main() { fi install_python_deps + + # Download leaflet-heat plugin for GSM heatmap (offline mode) + if [ ! -f "static/vendor/leaflet-heat/leaflet-heat.js" ]; then + info "Downloading leaflet-heat plugin..." + mkdir -p static/vendor/leaflet-heat + if curl -sL "https://unpkg.com/leaflet.heat@0.2.0/dist/leaflet-heat.js" \ + -o static/vendor/leaflet-heat/leaflet-heat.js; then + ok "leaflet-heat plugin downloaded" + else + warn "Failed to download leaflet-heat plugin. Heatmap will use CDN." + fi + fi + final_summary_and_hard_fail } diff --git a/static/vendor/leaflet-heat/leaflet-heat.js b/static/vendor/leaflet-heat/leaflet-heat.js new file mode 100644 index 0000000..aa8031a --- /dev/null +++ b/static/vendor/leaflet-heat/leaflet-heat.js @@ -0,0 +1,11 @@ +/* + (c) 2014, Vladimir Agafonkin + simpleheat, a tiny JavaScript library for drawing heatmaps with Canvas + https://github.com/mourner/simpleheat +*/ +!function(){"use strict";function t(i){return this instanceof t?(this._canvas=i="string"==typeof i?document.getElementById(i):i,this._ctx=i.getContext("2d"),this._width=i.width,this._height=i.height,this._max=1,void this.clear()):new t(i)}t.prototype={defaultRadius:25,defaultGradient:{.4:"blue",.6:"cyan",.7:"lime",.8:"yellow",1:"red"},data:function(t,i){return this._data=t,this},max:function(t){return this._max=t,this},add:function(t){return this._data.push(t),this},clear:function(){return this._data=[],this},radius:function(t,i){i=i||15;var a=this._circle=document.createElement("canvas"),s=a.getContext("2d"),e=this._r=t+i;return a.width=a.height=2*e,s.shadowOffsetX=s.shadowOffsetY=200,s.shadowBlur=i,s.shadowColor="black",s.beginPath(),s.arc(e-200,e-200,t,0,2*Math.PI,!0),s.closePath(),s.fill(),this},gradient:function(t){var i=document.createElement("canvas"),a=i.getContext("2d"),s=a.createLinearGradient(0,0,0,256);i.width=1,i.height=256;for(var e in t)s.addColorStop(e,t[e]);return a.fillStyle=s,a.fillRect(0,0,1,256),this._grad=a.getImageData(0,0,1,256).data,this},draw:function(t){this._circle||this.radius(this.defaultRadius),this._grad||this.gradient(this.defaultGradient);var i=this._ctx;i.clearRect(0,0,this._width,this._height);for(var a,s=0,e=this._data.length;e>s;s++)a=this._data[s],i.globalAlpha=Math.max(a[2]/this._max,t||.05),i.drawImage(this._circle,a[0]-this._r,a[1]-this._r);var n=i.getImageData(0,0,this._width,this._height);return this._colorize(n.data,this._grad),i.putImageData(n,0,0),this},_colorize:function(t,i){for(var a,s=3,e=t.length;e>s;s+=4)a=4*t[s],a&&(t[s-3]=i[a],t[s-2]=i[a+1],t[s-1]=i[a+2])}},window.simpleheat=t}(),/* + (c) 2014, Vladimir Agafonkin + Leaflet.heat, a tiny and fast heatmap plugin for Leaflet. + https://github.com/Leaflet/Leaflet.heat +*/ +L.HeatLayer=(L.Layer?L.Layer:L.Class).extend({initialize:function(t,i){this._latlngs=t,L.setOptions(this,i)},setLatLngs:function(t){return this._latlngs=t,this.redraw()},addLatLng:function(t){return this._latlngs.push(t),this.redraw()},setOptions:function(t){return L.setOptions(this,t),this._heat&&this._updateOptions(),this.redraw()},redraw:function(){return!this._heat||this._frame||this._map._animating||(this._frame=L.Util.requestAnimFrame(this._redraw,this)),this},onAdd:function(t){this._map=t,this._canvas||this._initCanvas(),t._panes.overlayPane.appendChild(this._canvas),t.on("moveend",this._reset,this),t.options.zoomAnimation&&L.Browser.any3d&&t.on("zoomanim",this._animateZoom,this),this._reset()},onRemove:function(t){t.getPanes().overlayPane.removeChild(this._canvas),t.off("moveend",this._reset,this),t.options.zoomAnimation&&t.off("zoomanim",this._animateZoom,this)},addTo:function(t){return t.addLayer(this),this},_initCanvas:function(){var t=this._canvas=L.DomUtil.create("canvas","leaflet-heatmap-layer leaflet-layer"),i=L.DomUtil.testProp(["transformOrigin","WebkitTransformOrigin","msTransformOrigin"]);t.style[i]="50% 50%";var a=this._map.getSize();t.width=a.x,t.height=a.y;var s=this._map.options.zoomAnimation&&L.Browser.any3d;L.DomUtil.addClass(t,"leaflet-zoom-"+(s?"animated":"hide")),this._heat=simpleheat(t),this._updateOptions()},_updateOptions:function(){this._heat.radius(this.options.radius||this._heat.defaultRadius,this.options.blur),this.options.gradient&&this._heat.gradient(this.options.gradient),this.options.max&&this._heat.max(this.options.max)},_reset:function(){var t=this._map.containerPointToLayerPoint([0,0]);L.DomUtil.setPosition(this._canvas,t);var i=this._map.getSize();this._heat._width!==i.x&&(this._canvas.width=this._heat._width=i.x),this._heat._height!==i.y&&(this._canvas.height=this._heat._height=i.y),this._redraw()},_redraw:function(){var t,i,a,s,e,n,h,o,r,d=[],_=this._heat._r,l=this._map.getSize(),m=new L.Bounds(L.point([-_,-_]),l.add([_,_])),c=void 0===this.options.max?1:this.options.max,u=void 0===this.options.maxZoom?this._map.getMaxZoom():this.options.maxZoom,f=1/Math.pow(2,Math.max(0,Math.min(u-this._map.getZoom(),12))),g=_/2,p=[],v=this._map._getMapPanePos(),w=v.x%g,y=v.y%g;for(t=0,i=this._latlngs.length;i>t;t++)if(a=this._map.latLngToContainerPoint(this._latlngs[t]),m.contains(a)){e=Math.floor((a.x-w)/g)+2,n=Math.floor((a.y-y)/g)+2;var x=void 0!==this._latlngs[t].alt?this._latlngs[t].alt:void 0!==this._latlngs[t][2]?+this._latlngs[t][2]:1;r=x*f,p[n]=p[n]||[],s=p[n][e],s?(s[0]=(s[0]*s[2]+a.x*r)/(s[2]+r),s[1]=(s[1]*s[2]+a.y*r)/(s[2]+r),s[2]+=r):p[n][e]=[a.x,a.y,r]}for(t=0,i=p.length;i>t;t++)if(p[t])for(h=0,o=p[t].length;o>h;h++)s=p[t][h],s&&d.push([Math.round(s[0]),Math.round(s[1]),Math.min(s[2],c)]);this._heat.data(d).draw(this.options.minOpacity),this._frame=null},_animateZoom:function(t){var i=this._map.getZoomScale(t.zoom),a=this._map._getCenterOffset(t.center)._multiplyBy(-i).subtract(this._map._getMapPanePos());L.DomUtil.setTransform?L.DomUtil.setTransform(this._canvas,a,i):this._canvas.style[L.DomUtil.TRANSFORM]=L.DomUtil.getTranslateString(a)+" scale("+i+")"}}),L.heatLayer=function(t,i){return new L.HeatLayer(t,i)}; \ No newline at end of file diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html index cfadbb3..a96e72c 100644 --- a/templates/gsm_spy_dashboard.html +++ b/templates/gsm_spy_dashboard.html @@ -18,6 +18,12 @@ {% endif %} + + {% if offline_settings.assets_source == 'local' %} + + {% else %} + + {% endif %} @@ -1450,6 +1456,9 @@ STANDBY
--:--:-- UTC
+ @@ -1805,6 +1814,11 @@ let isScanning = false; let currentRegion = 'americas'; + // Heatmap state + let heatmapLayer = null; + let heatmapVisible = false; + let heatmapLastRefresh = 0; + // Statistics let stats = { totalTowers: 0, @@ -2460,6 +2474,11 @@ stats.totalDevices = devicesCount; } updateStatsDisplay(); + + // Auto-refresh heatmap every 30s while visible + if (heatmapVisible && (Date.now() - heatmapLastRefresh > 30000)) { + updateHeatmap(); + } } function hideMonitorStatus() { @@ -3135,6 +3154,79 @@ } } + // ============================================ + // HEATMAP + // ============================================ + async function toggleHeatmap() { + if (heatmapVisible) { + removeHeatmap(); + } else { + await updateHeatmap(); + heatmapVisible = true; + const btn = document.getElementById('heatmapToggle'); + btn.textContent = 'Heatmap: ON'; + btn.style.borderColor = 'var(--accent-cyan)'; + btn.style.color = 'var(--accent-cyan)'; + } + } + + async function updateHeatmap() { + try { + const response = await fetch('/gsm_spy/crowd_density?hours=1'); + const data = await response.json(); + + if (!data || data.length === 0) return; + + const points = []; + let maxIntensity = 1; + + data.forEach(item => { + // Match CID to tower coordinates + const cid = item.cid; + for (const key in towers) { + const t = towers[key]; + if (t.cid === cid && t.lat && t.lon) { + const intensity = item.unique_devices || 1; + if (intensity > maxIntensity) maxIntensity = intensity; + points.push([t.lat, t.lon, intensity]); + break; + } + } + }); + + if (points.length === 0) return; + + // Remove existing layer before adding new one + if (heatmapLayer && gsmMap.hasLayer(heatmapLayer)) { + gsmMap.removeLayer(heatmapLayer); + } + + heatmapLayer = L.heatLayer(points, { + radius: 35, + blur: 25, + maxZoom: 17, + max: maxIntensity, + gradient: {0.2: '#304ffe', 0.4: '#00bcd4', 0.6: '#76ff03', 0.8: '#ffea00', 1.0: '#ff1744'} + }).addTo(gsmMap); + + heatmapLastRefresh = Date.now(); + } catch (error) { + console.error('[GSM SPY] Heatmap update error:', error); + } + } + + function removeHeatmap() { + if (heatmapLayer && gsmMap.hasLayer(heatmapLayer)) { + gsmMap.removeLayer(heatmapLayer); + } + heatmapLayer = null; + heatmapVisible = false; + const btn = document.getElementById('heatmapToggle'); + btn.textContent = 'Heatmap: OFF'; + btn.style.borderColor = ''; + btn.style.color = ''; + } + From e03ba3f5ed98b2a3c7a315b96765afc5549e1f9d Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 21:22:23 +0000 Subject: [PATCH 15/24] Fix heatmap: add type coercion, LAC matching, debug logging, and user feedback The heatmap silently failed when: CID types mismatched (string vs number), LAC wasn't checked (wrong tower matched), or no data existed yet (button showed ON with no layer). Now coerces CID/LAC to Number for comparison, validates coordinates with parseFloat, logs match diagnostics to console, and only shows ON when the layer is actually rendered. Co-Authored-By: Claude Opus 4.6 --- templates/gsm_spy_dashboard.html | 58 ++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html index a96e72c..82e0689 100644 --- a/templates/gsm_spy_dashboard.html +++ b/templates/gsm_spy_dashboard.html @@ -3161,12 +3161,14 @@ if (heatmapVisible) { removeHeatmap(); } else { - await updateHeatmap(); - heatmapVisible = true; - const btn = document.getElementById('heatmapToggle'); - btn.textContent = 'Heatmap: ON'; - btn.style.borderColor = 'var(--accent-cyan)'; - btn.style.color = 'var(--accent-cyan)'; + const success = await updateHeatmap(); + if (success) { + heatmapVisible = true; + const btn = document.getElementById('heatmapToggle'); + btn.textContent = 'Heatmap: ON'; + btn.style.borderColor = 'var(--accent-cyan)'; + btn.style.color = 'var(--accent-cyan)'; + } } } @@ -3175,26 +3177,52 @@ const response = await fetch('/gsm_spy/crowd_density?hours=1'); const data = await response.json(); - if (!data || data.length === 0) return; + console.log('[GSM SPY] Heatmap: crowd_density returned', Array.isArray(data) ? data.length + ' items' : data); + + if (!Array.isArray(data) || data.length === 0) { + if (!heatmapVisible) { + console.warn('[GSM SPY] Heatmap: no crowd density data. Start monitoring to collect device pings.'); + } + return false; + } const points = []; let maxIntensity = 1; + let matched = 0; + let unmatched = 0; data.forEach(item => { - // Match CID to tower coordinates - const cid = item.cid; + const itemCid = Number(item.cid); + const itemLac = Number(item.lac); + let found = false; + for (const key in towers) { const t = towers[key]; - if (t.cid === cid && t.lat && t.lon) { - const intensity = item.unique_devices || 1; - if (intensity > maxIntensity) maxIntensity = intensity; - points.push([t.lat, t.lon, intensity]); + if (Number(t.cid) === itemCid && Number(t.lac) === itemLac) { + if (t.lat && t.lon && !isNaN(parseFloat(t.lat)) && !isNaN(parseFloat(t.lon))) { + const intensity = item.unique_devices || 1; + if (intensity > maxIntensity) maxIntensity = intensity; + points.push([parseFloat(t.lat), parseFloat(t.lon), intensity]); + found = true; + } else { + console.log(`[GSM SPY] Heatmap: tower CID ${itemCid} LAC ${itemLac} has no coordinates yet`); + } break; } } + + if (found) matched++; + else unmatched++; }); - if (points.length === 0) return; + console.log(`[GSM SPY] Heatmap: ${matched} matched, ${unmatched} unmatched, ${points.length} points with coords`); + + if (points.length === 0) { + if (!heatmapVisible) { + console.warn('[GSM SPY] Heatmap: density data exists but no towers have coordinates. Wait for geocoding.'); + } + return false; + } // Remove existing layer before adding new one if (heatmapLayer && gsmMap.hasLayer(heatmapLayer)) { @@ -3210,8 +3238,10 @@ }).addTo(gsmMap); heatmapLastRefresh = Date.now(); + return true; } catch (error) { console.error('[GSM SPY] Heatmap update error:', error); + return false; } } From 0c656cff2b5de3938cb08808e24c5f79ca18bd91 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 21:27:36 +0000 Subject: [PATCH 16/24] Fix heatmap for towers with CID=0 or no geocoded coordinates The monitored tower may have CID=0 (partially decoded cell) which OpenCellID can't geocode, leaving it without coordinates. The heatmap now falls back through: monitored tower by ARFCN, any geocoded tower, then observer location. Also tracks the monitored ARFCN so the fallback can find the right tower even when CID matching fails. Co-Authored-By: Claude Opus 4.6 --- templates/gsm_spy_dashboard.html | 79 ++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html index 82e0689..85b12a2 100644 --- a/templates/gsm_spy_dashboard.html +++ b/templates/gsm_spy_dashboard.html @@ -1818,6 +1818,7 @@ let heatmapLayer = null; let heatmapVisible = false; let heatmapLastRefresh = 0; + let monitoredArfcn = null; // Statistics let stats = { @@ -2168,6 +2169,7 @@ updateScanStatus('Scan #' + data.scan + ' complete (' + data.towers_found + ' towers, ' + data.duration + 's)'); document.getElementById('scanProgressBar').style.width = '100%'; } else if (data.type === 'auto_monitor_started') { + monitoredArfcn = data.arfcn; showMonitorStatus(data.arfcn); console.log('[GSM SPY] Auto-monitor started on ARFCN', data.arfcn); } else if (data.type === 'monitor_heartbeat') { @@ -3172,6 +3174,48 @@ } } + function findTowerCoords(cid, lac) { + // Try exact CID+LAC match first + for (const key in towers) { + const t = towers[key]; + if (Number(t.cid) === cid && Number(t.lac) === lac && + t.lat && t.lon && !isNaN(parseFloat(t.lat)) && !isNaN(parseFloat(t.lon))) { + return [parseFloat(t.lat), parseFloat(t.lon)]; + } + } + return null; + } + + function findFallbackCoords() { + // 1) Try the monitored tower (by ARFCN) + if (monitoredArfcn !== null) { + for (const key in towers) { + const t = towers[key]; + if (Number(t.arfcn) === Number(monitoredArfcn) && + t.lat && t.lon && !isNaN(parseFloat(t.lat)) && !isNaN(parseFloat(t.lon))) { + console.log(`[GSM SPY] Heatmap: using monitored tower ARFCN ${monitoredArfcn} coords as fallback`); + return [parseFloat(t.lat), parseFloat(t.lon)]; + } + } + } + // 2) Try any geocoded tower + for (const key in towers) { + const t = towers[key]; + if (t.lat && t.lon && !isNaN(parseFloat(t.lat)) && !isNaN(parseFloat(t.lon))) { + console.log(`[GSM SPY] Heatmap: using tower CID ${t.cid} coords as fallback`); + return [parseFloat(t.lat), parseFloat(t.lon)]; + } + } + // 3) Fall back to observer location + const obsLat = parseFloat(document.getElementById('obsLat').value); + const obsLon = parseFloat(document.getElementById('obsLon').value); + if (!isNaN(obsLat) && !isNaN(obsLon)) { + console.log('[GSM SPY] Heatmap: using observer location as fallback'); + return [obsLat, obsLon]; + } + return null; + } + async function updateHeatmap() { try { const response = await fetch('/gsm_spy/crowd_density?hours=1'); @@ -3188,39 +3232,28 @@ const points = []; let maxIntensity = 1; - let matched = 0; - let unmatched = 0; + const fallbackCoords = findFallbackCoords(); data.forEach(item => { const itemCid = Number(item.cid); const itemLac = Number(item.lac); - let found = false; + const intensity = item.unique_devices || 1; + if (intensity > maxIntensity) maxIntensity = intensity; - for (const key in towers) { - const t = towers[key]; - if (Number(t.cid) === itemCid && Number(t.lac) === itemLac) { - if (t.lat && t.lon && !isNaN(parseFloat(t.lat)) && !isNaN(parseFloat(t.lon))) { - const intensity = item.unique_devices || 1; - if (intensity > maxIntensity) maxIntensity = intensity; - points.push([parseFloat(t.lat), parseFloat(t.lon), intensity]); - found = true; - } else { - console.log(`[GSM SPY] Heatmap: tower CID ${itemCid} LAC ${itemLac} has no coordinates yet`); - } - break; - } + // Try exact CID+LAC tower match + const coords = findTowerCoords(itemCid, itemLac); + if (coords) { + points.push([coords[0], coords[1], intensity]); + } else if (fallbackCoords) { + // Tower not geocoded or CID=0 — use fallback location + points.push([fallbackCoords[0], fallbackCoords[1], intensity]); } - - if (found) matched++; - else unmatched++; }); - console.log(`[GSM SPY] Heatmap: ${matched} matched, ${unmatched} unmatched, ${points.length} points with coords`); + console.log(`[GSM SPY] Heatmap: ${points.length} points from ${data.length} density records`); if (points.length === 0) { - if (!heatmapVisible) { - console.warn('[GSM SPY] Heatmap: density data exists but no towers have coordinates. Wait for geocoding.'); - } + console.warn('[GSM SPY] Heatmap: no plottable points (no tower coords and no fallback)'); return false; } From 2bed35dd640837cd5b18aa3b228d1ec9fb05840b Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 21:50:13 +0000 Subject: [PATCH 17/24] Fix signal handler deadlock and add satellite TLE data Remove logging and cleanup_all_processes() from signal handler to prevent deadlocks when another thread holds the logging or process lock. Process cleanup is handled by the atexit handler instead. Co-Authored-By: Claude Opus 4.6 --- gp.php | 210 +++++++++++++++++++++++++++++++++++++++++++++++ utils/process.py | 10 ++- 2 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 gp.php diff --git a/gp.php b/gp.php new file mode 100644 index 0000000..c901b45 --- /dev/null +++ b/gp.php @@ -0,0 +1,210 @@ +DMSP 5D-3 F16 (USA 172) +1 28054U 03048A 26037.66410905 .00000171 00000+0 11311-3 0 9991 +2 28054 99.0018 60.5544 0007736 150.6435 318.8272 14.14449870151032 +METEOSAT-9 (MSG-2) +1 28912U 05049B 26037.20698824 .00000122 00000+0 00000+0 0 9990 +2 28912 9.0646 55.4438 0001292 220.3216 340.7358 1.00280364 5681 +DMSP 5D-3 F17 (USA 191) +1 29522U 06050A 26037.63495522 .00000221 00000+0 13641-3 0 9997 +2 29522 98.7406 46.8646 0011088 71.3269 288.9107 14.14949568993957 +FENGYUN 3A +1 32958U 08026A 26037.29889977 .00000162 00000+0 97205-4 0 9995 +2 32958 98.6761 340.6748 0009336 139.4536 220.7337 14.19536323916838 +GOES 14 +1 35491U 09033A 26037.59737599 .00000128 00000+0 00000+0 0 9998 +2 35491 1.3510 84.7861 0001663 279.3774 203.6871 1.00112472 5283 +DMSP 5D-3 F18 (USA 210) +1 35951U 09057A 26037.59574243 .00000344 00000+0 20119-3 0 9997 +2 35951 98.8912 18.7405 0010014 262.2671 97.7365 14.14814612841124 +EWS-G2 (GOES 15) +1 36411U 10008A 26037.42417604 .00000037 00000+0 00000+0 0 9998 +2 36411 0.9477 85.6904 0004764 200.6178 64.5237 1.00275731 58322 +COMS 1 +1 36744U 10032A 26037.66884865 -.00000343 00000+0 00000+0 0 9998 +2 36744 4.4730 77.2684 0001088 239.9858 188.4845 1.00274368 49786 +FENGYUN 3B +1 37214U 10059A 26037.62488625 .00000510 00000+0 28715-3 0 9992 +2 37214 98.9821 82.9728 0021838 194.4193 280.6049 14.14810700788968 +SUOMI NPP +1 37849U 11061A 26037.58885771 .00000151 00000+0 92735-4 0 9993 +2 37849 98.7835 339.4455 0001677 23.1332 336.9919 14.19534335739918 +METEOSAT-10 (MSG-3) +1 38552U 12035B 26037.34062893 -.00000007 00000+0 00000+0 0 9993 +2 38552 4.3618 61.5789 0002324 286.1065 271.3938 1.00272839 49549 +METOP-B +1 38771U 12049A 26037.61376690 .00000161 00000+0 93652-4 0 9994 +2 38771 98.6708 91.6029 0002456 28.4142 331.7169 14.21434029694718 +INSAT-3D +1 39216U 13038B 26037.58021591 -.00000338 00000+0 00000+0 0 9998 +2 39216 1.5890 84.3012 0001719 220.0673 170.6954 1.00273812 45771 +FENGYUN 3C +1 39260U 13052A 26037.57879946 .00000181 00000+0 11337-3 0 9991 +2 39260 98.4839 17.5531 0015475 42.6626 317.5748 14.15718213640089 +METEOR-M 2 +1 40069U 14037A 26037.57010537 .00000364 00000+0 18579-3 0 9995 +2 40069 98.4979 18.0359 0006835 60.5067 299.6792 14.21415164600761 +HIMAWARI-8 +1 40267U 14060A 26037.58238259 -.00000273 00000+0 00000+0 0 9991 +2 40267 0.0457 252.0286 0000958 31.3580 203.5957 1.00278490 41450 +FENGYUN 2G +1 40367U 14090A 26037.64556289 -.00000299 00000+0 00000+0 0 9996 +2 40367 5.3089 74.4184 0001565 198.1345 195.9683 1.00263067 40698 +METEOSAT-11 (MSG-4) +1 40732U 15034A 26037.62779616 .00000065 00000+0 00000+0 0 9990 +2 40732 2.8728 71.8294 0001180 241.7344 58.8290 1.00268087 5909 +ELEKTRO-L 2 +1 41105U 15074A 26037.40900929 -.00000118 00000+0 00000+0 0 9998 +2 41105 6.3653 72.1489 0003612 229.0998 328.0297 1.00272232 37198 +INSAT-3DR +1 41752U 16054A 26037.65505200 -.00000075 00000+0 00000+0 0 9997 +2 41752 0.0554 93.8053 0013744 184.8269 167.9427 1.00271627 34504 +HIMAWARI-9 +1 41836U 16064A 26037.58238259 -.00000273 00000+0 00000+0 0 9990 +2 41836 0.0124 137.0088 0001068 210.1850 139.9064 1.00271322 33905 +GOES 16 +1 41866U 16071A 26037.60517604 -.00000089 00000+0 00000+0 0 9993 +2 41866 0.1490 94.1417 0002832 199.6896 316.0413 1.00271854 33798 +FENGYUN 4A +1 41882U 16077A 26037.65041625 -.00000356 00000+0 00000+0 0 9994 +2 41882 1.9907 81.7886 0006284 132.9819 279.8453 1.00276098 33627 +CYGFM05 +1 41884U 16078A 26037.42561482 .00027408 00000+0 46309-3 0 9992 +2 41884 34.9596 42.6579 0007295 332.2973 27.7361 15.50585086508404 +CYGFM04 +1 41885U 16078B 26037.34428483 .00032519 00000+0 49575-3 0 9994 +2 41885 34.9348 16.2836 0005718 359.2189 0.8525 15.53424088508589 +CYGFM02 +1 41886U 16078C 26037.35007768 .00035591 00000+0 50564-3 0 9998 +2 41886 34.9436 13.7490 0006836 2.8379 357.2383 15.55324468508720 +CYGFM01 +1 41887U 16078D 26037.39685921 .00028560 00000+0 47572-3 0 9999 +2 41887 34.9425 44.8029 0007415 323.1915 36.8298 15.50976884508344 +CYGFM08 +1 41888U 16078E 26037.34185185 .00031327 00000+0 49606-3 0 9997 +2 41888 34.9457 27.4597 0008083 350.5361 9.5208 15.52364941508578 +CYGFM07 +1 41890U 16078G 26037.32199955 .00032204 00000+0 49829-3 0 9990 +2 41890 34.9475 16.2411 0005914 7.0804 353.0002 15.53017084508593 +CYGFM03 +1 41891U 16078H 26037.35550653 .00031487 00000+0 48940-3 0 9995 +2 41891 34.9430 17.9804 0005939 349.1458 10.9136 15.52895386508574 +FENGYUN 3D +1 43010U 17072A 26037.62659924 .00000092 00000+0 65298-4 0 9990 +2 43010 98.9980 9.7978 0002479 69.6779 290.4663 14.19704535426460 +NOAA 20 (JPSS-1) +1 43013U 17073A 26037.60336371 .00000124 00000+0 79520-4 0 9999 +2 43013 98.7658 338.3064 0000377 14.6433 345.4754 14.19527655425942 +GOES 17 +1 43226U 18022A 26037.60794939 -.00000180 00000+0 00000+0 0 9993 +2 43226 0.6016 88.1527 0002754 213.0089 324.8756 1.00269924 29115 +FENGYUN 2H +1 43491U 18050A 26037.66161282 -.00000125 00000+0 00000+0 0 9992 +2 43491 2.6948 80.6967 0002145 171.8276 201.3055 1.00274855 28134 +METOP-C +1 43689U 18087A 26037.63948662 .00000167 00000+0 96262-4 0 9998 +2 43689 98.6834 99.5280 0001629 143.8933 216.2355 14.21510040376280 +GEO-KOMPSAT-2A +1 43823U 18100A 26037.57995591 .00000000 00000+0 00000+0 0 9996 +2 43823 0.0152 95.1913 0001141 313.4173 65.1318 1.00271011 26327 +METEOR-M2 2 +1 44387U 19038A 26037.58492015 .00000244 00000+0 12531-3 0 9993 +2 44387 98.9044 23.0180 0002141 55.2566 304.8814 14.24320728342700 +ARKTIKA-M 1 +1 47719U 21016A 26035.90384421 -.00000136 00000+0 00000+0 0 9994 +2 47719 63.1930 76.4940 7230705 269.3476 15.2984 2.00623094 36131 +FENGYUN 3E +1 49008U 21062A 26037.62586080 .00000245 00000+0 13631-3 0 9992 +2 49008 98.7499 42.4910 0002627 96.2819 263.8657 14.19890127238058 +GOES 18 +1 51850U 22021A 26037.59876267 .00000098 00000+0 00000+0 0 9999 +2 51850 0.0198 91.3546 0000843 290.2366 193.6737 1.00273310 5288 +NOAA 21 (JPSS-2) +1 54234U 22150A 26037.56792604 .00000152 00000+0 92800-4 0 9995 +2 54234 98.7521 338.1972 0001388 169.8161 190.3044 14.19543641168012 +METEOSAT-12 (MTG-I1) +1 54743U 22170C 26037.62580281 -.00000006 00000+0 00000+0 0 9990 +2 54743 0.7119 25.1556 0002027 273.4388 63.0828 1.00270670 11667 +TIANMU-1 03 +1 55973U 23039A 26037.63298084 .00025307 00000+0 57478-3 0 9994 +2 55973 97.5143 206.9374 0002852 198.5193 161.5950 15.43014921160671 +TIANMU-1 04 +1 55974U 23039B 26037.59957323 .00027172 00000+0 60888-3 0 9999 +2 55974 97.5075 206.0729 0003605 196.0743 164.0390 15.43399931160675 +TIANMU-1 05 +1 55975U 23039C 26037.60840428 .00024975 00000+0 56836-3 0 9995 +2 55975 97.5122 206.5750 0002421 224.3240 135.7814 15.42959696160653 +TIANMU-1 06 +1 55976U 23039D 26037.60004198 .00024821 00000+0 55598-3 0 9996 +2 55976 97.5133 207.0788 0002810 218.0193 142.0857 15.43432906160673 +FENGYUN 3G +1 56232U 23055A 26037.30935013 .00046475 00000+0 74423-3 0 9993 +2 56232 49.9940 300.8928 0009962 237.3703 122.6303 15.52544991159665 +METEOR-M2 3 +1 57166U 23091A 26037.62090481 .00000022 00000+0 28455-4 0 9999 +2 57166 98.6282 95.1607 0004003 174.5474 185.5750 14.24034408135931 +TIANMU-1 07 +1 57399U 23101A 26037.63242936 .00011510 00000+0 41012-3 0 9991 +2 57399 97.2786 91.2606 0002747 218.4597 141.6448 15.29074661141694 +TIANMU-1 08 +1 57400U 23101B 26037.66743594 .00011474 00000+0 41016-3 0 9996 +2 57400 97.2774 91.0783 0004440 227.8102 132.2762 15.28966110141699 +TIANMU-1 09 +1 57401U 23101C 26037.65072558 .00011360 00000+0 40433-3 0 9997 +2 57401 97.2732 90.5514 0003773 229.5297 130.5615 15.29113177141698 +TIANMU-1 10 +1 57402U 23101D 26037.61974057 .00011836 00000+0 42113-3 0 9994 +2 57402 97.2810 91.4302 0005461 233.7620 126.3116 15.29106286141685 +FENGYUN 3F +1 57490U 23111A 26037.61228373 .00000135 00000+0 84019-4 0 9997 +2 57490 98.6988 109.9815 0001494 99.6638 260.4707 14.19912110130332 +ARKTIKA-M 2 +1 58584U 23198A 26037.15964049 .00000160 00000+0 00000+0 0 9994 +2 58584 63.2225 168.8508 6872222 267.8808 18.8364 2.00612776 15698 +TIANMU-1 11 +1 58645U 23205A 26037.58628093 .00009545 00000+0 37951-3 0 9999 +2 58645 97.3574 61.2485 0010997 103.8713 256.3749 15.25445149117601 +TIANMU-1 12 +1 58646U 23205B 26037.61705312 .00010066 00000+0 40129-3 0 9995 +2 58646 97.3561 61.0663 0009308 89.8253 270.4052 15.25355570117590 +TIANMU-1 13 +1 58647U 23205C 26037.64894829 .00010029 00000+0 39925-3 0 9992 +2 58647 97.3589 61.3229 0009456 74.8265 285.4018 15.25403883117592 +TIANMU-1 14 +1 58648U 23205D 26037.63305929 .00009719 00000+0 38718-3 0 9993 +2 58648 97.3523 60.6045 0010314 77.9995 282.2399 15.25381326117592 +TIANMU-1 19 +1 58660U 23208A 26037.58812600 .00016491 00000+0 58449-3 0 9991 +2 58660 97.4377 153.5627 0006125 66.0574 294.1307 15.29155961117352 +TIANMU-1 20 +1 58661U 23208B 26037.59661536 .00016638 00000+0 56823-3 0 9990 +2 58661 97.4315 154.0738 0008420 72.4906 287.7255 15.30347593117439 +TIANMU-1 21 +1 58662U 23208C 26037.56944589 .00017161 00000+0 55253-3 0 9998 +2 58662 97.4367 156.2063 0008160 67.8039 292.4068 15.32247056117540 +TIANMU-1 22 +1 58663U 23208D 26037.59847459 .00015396 00000+0 55169-3 0 9994 +2 58663 97.4371 153.6033 0005010 87.2275 272.9538 15.28818503117364 +TIANMU-1 15 +1 58700U 24004A 26037.63062994 .00009739 00000+0 38850-3 0 9991 +2 58700 97.4651 223.9243 0008449 88.7599 271.4607 15.25356935115862 +TIANMU-1 16 +1 58701U 24004B 26037.61474986 .00010691 00000+0 42590-3 0 9993 +2 58701 97.4590 223.2544 0006831 91.0928 269.1093 15.25387104115863 +TIANMU-1 17 +1 58702U 24004C 26037.59783649 .00011079 00000+0 44078-3 0 9994 +2 58702 97.4624 223.6760 0006020 92.0871 268.1056 15.25425175115852 +TIANMU-1 18 +1 58703U 24004D 26037.64767373 .00010786 00000+0 42976-3 0 9996 +2 58703 97.4642 223.9320 0005432 91.0134 269.1726 15.25387870115860 +INSAT-3DS +1 58990U 24033A 26037.64159978 -.00000153 00000+0 00000+0 0 9998 +2 58990 0.0277 242.2492 0001855 99.2205 108.3003 1.00271452 45758 +METEOR-M2 4 +1 59051U 24039A 26037.62796654 .00000070 00000+0 51194-4 0 9991 +2 59051 98.6849 358.6843 0006923 178.9165 181.2029 14.22412185100701 +GOES 19 +1 60133U 24119A 26037.61098274 -.00000246 00000+0 00000+0 0 9996 +2 60133 0.0027 288.6290 0001204 74.2636 278.5881 1.00270967 5651 +FENGYUN 3H +1 65815U 25219A 26037.60879211 .00000151 00000+0 91464-4 0 9990 +2 65815 98.6649 341.0050 0001596 86.5100 273.6260 14.19924132 18857 diff --git a/utils/process.py b/utils/process.py index 202f370..79a5796 100644 --- a/utils/process.py +++ b/utils/process.py @@ -85,11 +85,13 @@ atexit.register(cleanup_all_processes) # Handle signals for graceful shutdown def _signal_handler(signum, frame): - """Handle termination signals.""" + """Handle termination signals. + + Keep this minimal — logging and lock acquisition in signal handlers + can deadlock when another thread holds the logging or process lock. + Process cleanup is handled by the atexit handler registered above. + """ import sys - logger.info(f"Received signal {signum}, cleaning up...") - cleanup_all_processes() - # Re-raise KeyboardInterrupt for SIGINT so Flask can handle shutdown if signum == signal.SIGINT: raise KeyboardInterrupt() sys.exit(0) From c2891938ab33bfdae4799eef7d9b23bf92fd1590 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 22:04:12 +0000 Subject: [PATCH 18/24] Remove GSM spy functionality for legal compliance Remove all GSM cellular intelligence features including tower scanning, signal monitoring, rogue detection, crowd density analysis, and OpenCellID integration across routes, templates, utils, tests, and build configuration. Co-Authored-By: Claude Opus 4.6 --- Dockerfile | 28 - app.py | 59 +- config.py | 31 +- routes/__init__.py | 2 - routes/gsm_spy.py | 2051 --------------- setup.sh | 127 +- static/js/core/settings-manager.js | 55 +- templates/gsm_spy_dashboard.html | 3300 ------------------------ templates/index.html | 4 - templates/partials/nav.html | 1 - templates/partials/settings-modal.html | 65 - tests/test_gsm_spy.py | 360 --- utils/constants.py | 10 - utils/database.py | 185 -- utils/dependencies.py | 32 - utils/gsm_geocoding.py | 226 -- utils/logging.py | 1 - 17 files changed, 21 insertions(+), 6516 deletions(-) delete mode 100644 routes/gsm_spy.py delete mode 100644 templates/gsm_spy_dashboard.html delete mode 100644 tests/test_gsm_spy.py delete mode 100644 utils/gsm_geocoding.py diff --git a/Dockerfile b/Dockerfile index a435d81..0b45b00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,39 +48,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ airspy \ limesuite \ hackrf \ - # GSM Intelligence (tshark for packet parsing) - tshark \ # Utilities curl \ procps \ && rm -rf /var/lib/apt/lists/* -# GSM Intelligence: gr-gsm (grgsm_scanner, grgsm_livemon) -# Install from apt if available, otherwise build from source -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - gnuradio gr-osmosdr gr-gsm 2>/dev/null \ - || ( \ - apt-get install -y --no-install-recommends \ - gnuradio gnuradio-dev gr-osmosdr \ - git cmake libboost-all-dev libcppunit-dev swig \ - doxygen liblog4cpp5-dev python3-scipy python3-numpy \ - libvolk-dev libfftw3-dev build-essential \ - && cd /tmp \ - && git clone --depth 1 https://github.com/bkerler/gr-gsm.git \ - && cd gr-gsm \ - && mkdir build && cd build \ - && cmake .. \ - && make -j$(nproc) \ - && make install \ - && ldconfig \ - && rm -rf /tmp/gr-gsm \ - && apt-get remove -y gnuradio-dev libcppunit-dev swig doxygen \ - liblog4cpp5-dev libvolk-dev build-essential git cmake \ - && apt-get autoremove -y \ - ) \ - && rm -rf /var/lib/apt/lists/* - # Build dump1090-fa and acarsdec from source (packages not available in slim repos) RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ diff --git a/app.py b/app.py index 76ee903..bb91402 100644 --- a/app.py +++ b/app.py @@ -39,7 +39,6 @@ from utils.constants import ( MAX_VESSEL_AGE_SECONDS, MAX_DSC_MESSAGE_AGE_SECONDS, MAX_DEAUTH_ALERTS_AGE_SECONDS, - MAX_GSM_AGE_SECONDS, QUEUE_MAX_SIZE, ) import logging @@ -188,16 +187,6 @@ deauth_detector = None deauth_detector_queue = queue.Queue(maxsize=QUEUE_MAX_SIZE) deauth_detector_lock = threading.Lock() -# GSM Spy -gsm_spy_scanner_running = False # Flag: scanner thread active -gsm_spy_livemon_process = None # For grgsm_livemon process -gsm_spy_monitor_process = None # For tshark monitoring process -gsm_spy_queue = queue.Queue(maxsize=QUEUE_MAX_SIZE) -gsm_spy_lock = threading.Lock() -gsm_spy_active_device = None -gsm_spy_selected_arfcn = None -gsm_spy_region = 'Americas' # Default band - # ============================================ # GLOBAL STATE DICTIONARIES # ============================================ @@ -230,16 +219,6 @@ dsc_messages = DataStore(max_age_seconds=MAX_DSC_MESSAGE_AGE_SECONDS, name='dsc_ # Deauth alerts - using DataStore for automatic cleanup deauth_alerts = DataStore(max_age_seconds=MAX_DEAUTH_ALERTS_AGE_SECONDS, name='deauth_alerts') -# GSM Spy data stores -gsm_spy_towers = DataStore( - max_age_seconds=MAX_GSM_AGE_SECONDS, - name='gsm_spy_towers' -) -gsm_spy_devices = DataStore( - max_age_seconds=MAX_GSM_AGE_SECONDS, - name='gsm_spy_devices' -) - # Satellite state satellite_passes = [] # Predicted satellite passes (not auto-cleaned, calculated) @@ -252,8 +231,6 @@ cleanup_manager.register(adsb_aircraft) cleanup_manager.register(ais_vessels) cleanup_manager.register(dsc_messages) cleanup_manager.register(deauth_alerts) -cleanup_manager.register(gsm_spy_towers) -cleanup_manager.register(gsm_spy_devices) # ============================================ # SDR DEVICE REGISTRY @@ -687,8 +664,6 @@ def kill_all() -> Response: global current_process, sensor_process, wifi_process, adsb_process, ais_process, acars_process global aprs_process, aprs_rtl_process, dsc_process, dsc_rtl_process, bt_process global dmr_process, dmr_rtl_process - global gsm_spy_livemon_process, gsm_spy_monitor_process - global gsm_spy_scanner_running, gsm_spy_active_device, gsm_spy_selected_arfcn, gsm_spy_region # Import adsb and ais modules to reset their state from routes import adsb as adsb_module @@ -701,8 +676,7 @@ def kill_all() -> Response: 'airodump-ng', 'aireplay-ng', 'airmon-ng', 'dump1090', 'acarsdec', 'direwolf', 'AIS-catcher', 'hcitool', 'bluetoothctl', 'dsd', - 'rtl_tcp', 'rtl_power', 'rtlamr', 'ffmpeg', - 'grgsm_scanner', 'grgsm_livemon', 'tshark' + 'rtl_tcp', 'rtl_power', 'rtlamr', 'ffmpeg' ] for proc in processes_to_kill: @@ -771,29 +745,6 @@ def kill_all() -> Response: except Exception: pass - # Reset GSM Spy state - with gsm_spy_lock: - gsm_spy_scanner_running = False - gsm_spy_active_device = None - gsm_spy_selected_arfcn = None - gsm_spy_region = 'Americas' - - if gsm_spy_livemon_process: - try: - if safe_terminate(gsm_spy_livemon_process): - killed.append('grgsm_livemon') - except Exception: - pass - gsm_spy_livemon_process = None - - if gsm_spy_monitor_process: - try: - if safe_terminate(gsm_spy_monitor_process): - killed.append('tshark') - except Exception: - pass - gsm_spy_monitor_process = None - # Clear SDR device registry with sdr_device_registry_lock: sdr_device_registry.clear() @@ -885,19 +836,11 @@ def main() -> None: # Register database cleanup functions from utils.database import ( - cleanup_old_gsm_signals, - cleanup_old_gsm_tmsi_log, - cleanup_old_gsm_velocity_log, cleanup_old_signal_history, cleanup_old_timeline_entries, cleanup_old_dsc_alerts, cleanup_old_payloads ) - # GSM cleanups: signals (60 days), TMSI log (24 hours), velocity (1 hour) - # Interval multiplier: cleanup every N cycles (60s interval = 1 cleanup per hour at multiplier 60) - cleanup_manager.register_db_cleanup(cleanup_old_gsm_tmsi_log, interval_multiplier=60) # Every hour - cleanup_manager.register_db_cleanup(cleanup_old_gsm_velocity_log, interval_multiplier=60) # Every hour - cleanup_manager.register_db_cleanup(cleanup_old_gsm_signals, interval_multiplier=1440) # Every 24 hours cleanup_manager.register_db_cleanup(cleanup_old_signal_history, interval_multiplier=1440) # Every 24 hours cleanup_manager.register_db_cleanup(cleanup_old_timeline_entries, interval_multiplier=1440) # Every 24 hours cleanup_manager.register_db_cleanup(cleanup_old_dsc_alerts, interval_multiplier=1440) # Every 24 hours diff --git a/config.py b/config.py index 98f2d1d..2e01d48 100644 --- a/config.py +++ b/config.py @@ -204,25 +204,20 @@ SATELLITE_UPDATE_INTERVAL = _get_env_int('SATELLITE_UPDATE_INTERVAL', 30) SATELLITE_TRAJECTORY_POINTS = _get_env_int('SATELLITE_TRAJECTORY_POINTS', 30) SATELLITE_ORBIT_MINUTES = _get_env_int('SATELLITE_ORBIT_MINUTES', 45) -# Update checking -GITHUB_REPO = _get_env('GITHUB_REPO', 'smittix/intercept') -UPDATE_CHECK_ENABLED = _get_env_bool('UPDATE_CHECK_ENABLED', True) -UPDATE_CHECK_INTERVAL_HOURS = _get_env_int('UPDATE_CHECK_INTERVAL_HOURS', 6) - -# Alerting -ALERT_WEBHOOK_URL = _get_env('ALERT_WEBHOOK_URL', '') -ALERT_WEBHOOK_SECRET = _get_env('ALERT_WEBHOOK_SECRET', '') -ALERT_WEBHOOK_TIMEOUT = _get_env_int('ALERT_WEBHOOK_TIMEOUT', 5) - -# Admin credentials -ADMIN_USERNAME = _get_env('ADMIN_USERNAME', 'admin') -ADMIN_PASSWORD = _get_env('ADMIN_PASSWORD', 'admin') +# Update checking +GITHUB_REPO = _get_env('GITHUB_REPO', 'smittix/intercept') +UPDATE_CHECK_ENABLED = _get_env_bool('UPDATE_CHECK_ENABLED', True) +UPDATE_CHECK_INTERVAL_HOURS = _get_env_int('UPDATE_CHECK_INTERVAL_HOURS', 6) + +# Alerting +ALERT_WEBHOOK_URL = _get_env('ALERT_WEBHOOK_URL', '') +ALERT_WEBHOOK_SECRET = _get_env('ALERT_WEBHOOK_SECRET', '') +ALERT_WEBHOOK_TIMEOUT = _get_env_int('ALERT_WEBHOOK_TIMEOUT', 5) + +# Admin credentials +ADMIN_USERNAME = _get_env('ADMIN_USERNAME', 'admin') +ADMIN_PASSWORD = _get_env('ADMIN_PASSWORD', 'admin') -# GSM Spy settings -GSM_OPENCELLID_API_KEY = _get_env('GSM_OPENCELLID_API_KEY', '') -GSM_OPENCELLID_API_URL = _get_env('GSM_OPENCELLID_API_URL', 'https://opencellid.org/cell/get') -GSM_API_DAILY_LIMIT = _get_env_int('GSM_API_DAILY_LIMIT', 1000) -GSM_TA_METERS_PER_UNIT = _get_env_int('GSM_TA_METERS_PER_UNIT', 554) def configure_logging() -> None: """Configure application logging.""" diff --git a/routes/__init__.py b/routes/__init__.py index af211cc..0ac1fc2 100644 --- a/routes/__init__.py +++ b/routes/__init__.py @@ -31,7 +31,6 @@ def register_blueprints(app): from .websdr import websdr_bp from .alerts import alerts_bp from .recordings import recordings_bp - from .gsm_spy import gsm_spy_bp app.register_blueprint(pager_bp) app.register_blueprint(sensor_bp) @@ -62,7 +61,6 @@ def register_blueprints(app): app.register_blueprint(websdr_bp) # HF/Shortwave WebSDR app.register_blueprint(alerts_bp) # Cross-mode alerts app.register_blueprint(recordings_bp) # Session recordings - app.register_blueprint(gsm_spy_bp) # GSM cellular intelligence # Initialize TSCM state with queue and lock from app import app as app_module diff --git a/routes/gsm_spy.py b/routes/gsm_spy.py deleted file mode 100644 index 49aca53..0000000 --- a/routes/gsm_spy.py +++ /dev/null @@ -1,2051 +0,0 @@ -"""GSM Spy route handlers for cellular tower and device tracking.""" - -from __future__ import annotations - -import json -import logging -import os -import queue -import re -import shutil -import subprocess -import threading -import time -from datetime import datetime, timedelta -from typing import Any - -import requests -from flask import Blueprint, Response, jsonify, render_template, request - -import app as app_module -import config -from config import SHARED_OBSERVER_LOCATION_ENABLED -from utils.database import get_db -from utils.process import register_process, safe_terminate, unregister_process -from utils.sse import format_sse -from utils.validation import validate_device_index - -from utils.logging import get_logger -logger = get_logger('intercept.gsm_spy') -logger.setLevel(logging.DEBUG) # GSM Spy needs verbose logging for diagnostics - -gsm_spy_bp = Blueprint('gsm_spy', __name__, url_prefix='/gsm_spy') - -# Regional band configurations (G-01) -REGIONAL_BANDS = { - 'Americas': { - 'GSM850': {'start': 869e6, 'end': 894e6, 'arfcn_start': 128, 'arfcn_end': 251}, - 'PCS1900': {'start': 1930e6, 'end': 1990e6, 'arfcn_start': 512, 'arfcn_end': 810} - }, - 'Europe': { - 'GSM800': {'start': 832e6, 'end': 862e6, 'arfcn_start': 438, 'arfcn_end': 511}, # E-GSM800 downlink - 'GSM850': {'start': 869e6, 'end': 894e6, 'arfcn_start': 128, 'arfcn_end': 251}, # Also used in some EU countries - 'EGSM900': {'start': 935e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, # DL = 935 + 0.2*ARFCN - 'EGSM900_EXT': {'start': 925.2e6, 'end': 935e6, 'arfcn_start': 975, 'arfcn_end': 1023}, # E-GSM extension - 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} - }, - 'Asia': { - 'EGSM900': {'start': 935e6, 'end': 960e6, 'arfcn_start': 0, 'arfcn_end': 124}, # DL = 935 + 0.2*ARFCN - 'EGSM900_EXT': {'start': 925.2e6, 'end': 935e6, 'arfcn_start': 975, 'arfcn_end': 1023}, # E-GSM extension - 'DCS1800': {'start': 1805e6, 'end': 1880e6, 'arfcn_start': 512, 'arfcn_end': 885} - } -} - -# Module state tracking -gsm_using_service = False -gsm_connected = False -gsm_towers_found = 0 -gsm_devices_tracked = 0 - -# Geocoding worker state -_geocoding_worker_thread = None - - -# ============================================ -# API Usage Tracking Helper Functions -# ============================================ - -def get_api_usage_today(): - """Get OpenCellID API usage count for today.""" - from utils.database import get_setting - today = datetime.now().date().isoformat() - usage_date = get_setting('gsm.opencellid.usage_date', '') - - # Reset counter if new day - if usage_date != today: - from utils.database import set_setting - set_setting('gsm.opencellid.usage_date', today) - set_setting('gsm.opencellid.usage_count', 0) - return 0 - - return get_setting('gsm.opencellid.usage_count', 0) - - -def increment_api_usage(): - """Increment OpenCellID API usage counter.""" - from utils.database import set_setting - current = get_api_usage_today() - set_setting('gsm.opencellid.usage_count', current + 1) - return current + 1 - - -def get_opencellid_api_key(): - """Get OpenCellID API key, checking env var first, then database setting.""" - env_key = config.GSM_OPENCELLID_API_KEY - if env_key: - return env_key - from utils.database import get_setting - return get_setting('gsm.opencellid.api_key', '') - - -def can_use_api(): - """Check if we can make an API call within daily limit.""" - if not get_opencellid_api_key(): - return False - current_usage = get_api_usage_today() - return current_usage < config.GSM_API_DAILY_LIMIT - - -# ============================================ -# Background Geocoding Worker -# ============================================ - -def start_geocoding_worker(): - """Start background thread for async geocoding.""" - global _geocoding_worker_thread - - # Clean poisoned cache entries (rows with NULL lat/lon from failed API responses) - try: - with get_db() as conn: - deleted = conn.execute( - 'DELETE FROM gsm_cells WHERE lat IS NULL OR lon IS NULL' - ).rowcount - conn.commit() - if deleted: - logger.info(f"Cleaned {deleted} poisoned cache entries (NULL coordinates)") - except Exception as e: - logger.warning(f"Could not clean cache: {e}") - - if _geocoding_worker_thread is None or not _geocoding_worker_thread.is_alive(): - _geocoding_worker_thread = threading.Thread( - target=geocoding_worker, - daemon=True, - name='gsm-geocoding-worker' - ) - _geocoding_worker_thread.start() - logger.info("Started geocoding worker thread") - - -def geocoding_worker(): - """Worker thread processes pending geocoding requests.""" - from utils.gsm_geocoding import lookup_cell_from_api, get_geocoding_queue - - geocoding_queue = get_geocoding_queue() - - while True: - try: - # Wait for pending tower with timeout - tower_data = geocoding_queue.get(timeout=5) - - # Check API key and rate limit - if not can_use_api(): - if not get_opencellid_api_key(): - logger.warning("OpenCellID API key not configured (set INTERCEPT_GSM_OPENCELLID_API_KEY or configure in Settings > API Keys)") - else: - current_usage = get_api_usage_today() - logger.warning(f"OpenCellID API daily limit reached ({current_usage}/{config.GSM_API_DAILY_LIMIT})") - geocoding_queue.task_done() - continue - - # Call API - mcc = tower_data.get('mcc') - mnc = tower_data.get('mnc') - lac = tower_data.get('lac') - cid = tower_data.get('cid') - - logger.debug(f"Geocoding tower via API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - - coords = lookup_cell_from_api(mcc, mnc, lac, cid) - - if coords: - # Update tower data with coordinates - tower_data['lat'] = coords['lat'] - tower_data['lon'] = coords['lon'] - tower_data['source'] = 'api' - tower_data['status'] = 'resolved' - tower_data['type'] = 'tower_update' - - # Add optional fields if available - if coords.get('azimuth') is not None: - tower_data['azimuth'] = coords['azimuth'] - if coords.get('range_meters') is not None: - tower_data['range_meters'] = coords['range_meters'] - if coords.get('operator'): - tower_data['operator'] = coords['operator'] - if coords.get('radio'): - tower_data['radio'] = coords['radio'] - - # Update DataStore - key = f"{mcc}_{mnc}_{lac}_{cid}" - app_module.gsm_spy_towers[key] = tower_data - - # Send update to SSE stream - try: - app_module.gsm_spy_queue.put_nowait(tower_data) - logger.info(f"Resolved coordinates for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - except queue.Full: - logger.warning("SSE queue full, dropping tower update") - - # Increment API usage counter - usage_count = increment_api_usage() - logger.info(f"OpenCellID API call #{usage_count} today") - - else: - logger.warning(f"Could not resolve coordinates for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - - geocoding_queue.task_done() - - # Rate limiting between API calls (be nice to OpenCellID) - time.sleep(1) - - except queue.Empty: - # No pending towers, continue waiting - continue - except Exception as e: - logger.error(f"Geocoding worker error: {e}", exc_info=True) - time.sleep(1) - - -def arfcn_to_frequency(arfcn): - """Convert ARFCN to downlink frequency in Hz. - - Uses REGIONAL_BANDS to determine the correct band and conversion formula. - Returns frequency in Hz (e.g., 925800000 for 925.8 MHz). - """ - arfcn = int(arfcn) - - # Search all bands to find which one this ARFCN belongs to - for region_bands in REGIONAL_BANDS.values(): - for band_name, band_info in region_bands.items(): - arfcn_start = band_info['arfcn_start'] - arfcn_end = band_info['arfcn_end'] - - if arfcn_start <= arfcn <= arfcn_end: - # Found the right band, calculate frequency - # Downlink frequency = band_start + (arfcn - arfcn_start) * 200kHz - freq_hz = band_info['start'] + (arfcn - arfcn_start) * 200000 - return int(freq_hz) - - # If ARFCN not found in any band, raise error - raise ValueError(f"ARFCN {arfcn} not found in any known GSM band") - - -def validate_band_names(bands: list[str], region: str) -> tuple[list[str], str | None]: - """Validate band names against REGIONAL_BANDS whitelist. - - Args: - bands: List of band names from user input - region: Region name (Americas, Europe, Asia) - - Returns: - Tuple of (validated_bands, error_message) - """ - if not bands: - return [], None - - region_bands = REGIONAL_BANDS.get(region) - if not region_bands: - return [], f"Invalid region: {region}" - - valid_band_names = set(region_bands.keys()) - invalid_bands = [b for b in bands if b not in valid_band_names] - - if invalid_bands: - return [], (f"Invalid bands for {region}: {', '.join(invalid_bands)}. " - f"Valid bands: {', '.join(sorted(valid_band_names))}") - - return bands, None - - -# tshark field name discovery - field names vary between Wireshark versions -_tshark_fields_cache: dict[str, str] | None = None - - -def _discover_tshark_fields() -> dict[str, str | None]: - """Discover correct tshark field names for GSM A protocol. - - Searches tshark's registered fields for keywords to find the actual - names used by the installed Wireshark version, then validates them. - - Returns: - Dict mapping logical names to actual tshark field names (or None): - {'ta': ..., 'tmsi': ..., 'imsi': ..., 'lac': ..., 'cid': ...} - """ - global _tshark_fields_cache - if _tshark_fields_cache is not None: - return _tshark_fields_cache - - # Search patterns for each logical field (applied to tshark -G fields output) - # Each entry: (keyword_patterns, exclusion_patterns) - # We search the field filter name column for matches - search_config = { - 'ta': { - 'keywords': ['timing_adv', 'timing_advance'], - 'prefer': ['gsm_a'], # prefer GSM A protocol fields - }, - 'tmsi': { - 'keywords': ['.tmsi'], - 'prefer': ['gsm_a'], - }, - 'imsi': { - 'keywords': ['.imsi'], - 'prefer': ['e212', 'gsm_a'], - }, - 'lac': { - 'keywords': ['.lac'], - 'prefer': ['gsm_a', 'e212'], - }, - 'cid': { - 'keywords': ['cellid', 'cell_ci', '.cell_id', 'e212.ci'], - 'prefer': ['gsm_a', 'e212'], - }, - } - - # Step 1: Get all field names from tshark (F lines only, not P protocol lines) - all_field_names = [] - try: - result = subprocess.run( - ['tshark', '-G', 'fields'], - capture_output=True, text=True, timeout=15 - ) - for line in result.stdout.splitlines(): - if not line.startswith('F\t'): - continue # Only actual fields, not protocols - parts = line.split('\t') - if len(parts) >= 3: - all_field_names.append(parts[2]) - except Exception as e: - logger.warning(f"Could not query tshark fields: {e}") - - # Step 2: Search for candidate fields matching each logical name - candidates: dict[str, list[str]] = {} - for logical_name, config in search_config.items(): - matches = [] - for field_name in all_field_names: - for keyword in config['keywords']: - if keyword in field_name: - matches.append(field_name) - break - # Sort: preferred protocol prefixes first - def sort_key(name): - for i, pref in enumerate(config['prefer']): - if name.startswith(pref): - return i - return 100 - matches.sort(key=sort_key) - candidates[logical_name] = matches - - logger.info(f"tshark field candidates from -G fields: {candidates}") - - # Step 3: Validate candidates by testing with tshark -r /dev/null - # Collect all unique candidate names - all_candidate_names = set() - for field_list in candidates.values(): - all_candidate_names.update(field_list) - - valid_fields = set() - if all_candidate_names: - # Test in batches to identify which are valid - # Test each individually since batch testing makes it hard to identify valid ones - for field_name in all_candidate_names: - try: - result = subprocess.run( - ['tshark', '-T', 'fields', '-e', field_name, '-r', '/dev/null'], - capture_output=True, text=True, timeout=5 - ) - if result.returncode == 0 or 'aren\'t valid' not in result.stderr: - valid_fields.add(field_name) - except Exception: - pass - - logger.info(f"Validated tshark -e fields: {sorted(valid_fields)}") - - # Step 4: Resolve each logical field to the first valid candidate - resolved: dict[str, str | None] = {} - for logical_name, field_candidates in candidates.items(): - resolved[logical_name] = None - for candidate in field_candidates: - if candidate in valid_fields: - resolved[logical_name] = candidate - break - if resolved[logical_name] is None and field_candidates: - logger.warning( - f"No valid tshark field for '{logical_name}'. " - f"Candidates were: {field_candidates}" - ) - - logger.info(f"Resolved tshark fields: {resolved}") - _tshark_fields_cache = resolved - return resolved - - -def _start_monitoring_processes(arfcn: int, device_index: int) -> tuple[subprocess.Popen, subprocess.Popen, list[str]]: - """Start grgsm_livemon and tshark processes for monitoring an ARFCN. - - Returns: - Tuple of (grgsm_process, tshark_process, field_order) - field_order is the list of logical field names in tshark column order. - - Raises: - FileNotFoundError: If grgsm_livemon or tshark not found - RuntimeError: If grgsm_livemon or tshark exits immediately - """ - frequency_hz = arfcn_to_frequency(arfcn) - frequency_mhz = frequency_hz / 1e6 - - # Check prerequisites - if not shutil.which('grgsm_livemon'): - raise FileNotFoundError('grgsm_livemon not found. Please install gr-gsm.') - - # Start grgsm_livemon - grgsm_cmd = [ - 'grgsm_livemon', - '--args', f'rtl={device_index}', - '-f', f'{frequency_mhz}M' - ] - env = dict(os.environ, - OSMO_FSM_DUP_CHECK_DISABLED='1', - PYTHONUNBUFFERED='1', - QT_QPA_PLATFORM='offscreen') - logger.info(f"Starting grgsm_livemon: {' '.join(grgsm_cmd)}") - grgsm_proc = subprocess.Popen( - grgsm_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=env - ) - register_process(grgsm_proc) - logger.info(f"Started grgsm_livemon (PID: {grgsm_proc.pid})") - - # Wait and check it didn't die immediately - time.sleep(2) - - if grgsm_proc.poll() is not None: - # Process already exited - capture stderr for diagnostics - stderr_output = '' - try: - stderr_output = grgsm_proc.stderr.read() - except Exception: - pass - exit_code = grgsm_proc.returncode - logger.error( - f"grgsm_livemon exited immediately (code: {exit_code}). " - f"stderr: {stderr_output[:500]}" - ) - unregister_process(grgsm_proc) - raise RuntimeError( - f'grgsm_livemon failed (exit code {exit_code}): {stderr_output[:200]}' - ) - - # Start stderr reader thread for grgsm_livemon diagnostics - def read_livemon_stderr(): - try: - for line in iter(grgsm_proc.stderr.readline, ''): - if line: - logger.debug(f"grgsm_livemon stderr: {line.strip()}") - except Exception: - pass - threading.Thread(target=read_livemon_stderr, daemon=True).start() - - # Start tshark - if not shutil.which('tshark'): - safe_terminate(grgsm_proc) - unregister_process(grgsm_proc) - raise FileNotFoundError('tshark not found. Please install wireshark/tshark.') - - fields = _discover_tshark_fields() - - # Build field list from only valid (non-None) fields - # Track order so parser knows which column is which - field_order = [] # list of logical names in column order - tshark_cmd = [ - 'tshark', - '-i', 'lo', - '-l', # Line-buffered output for live capture - '-f', 'udp port 4729', # Capture filter: only GSMTAP packets - ] - - # No display filter (-Y) — the capture filter (-f 'udp port 4729') - # already limits to GSMTAP packets, and the parser discards rows - # without TMSI/IMSI. A -Y filter on gsm_a.tmsi misses paging - # requests where the TMSI lives under a different field path. - tshark_cmd.extend(['-T', 'fields']) - - # Add -e for each available field in known order - for logical_name in ['ta', 'tmsi', 'imsi', 'lac', 'cid']: - if fields.get(logical_name): - tshark_cmd.extend(['-e', fields[logical_name]]) - field_order.append(logical_name) - - if not field_order: - safe_terminate(grgsm_proc) - unregister_process(grgsm_proc) - raise RuntimeError('No valid tshark fields found for GSM capture') - - logger.info(f"tshark field order: {field_order}") - logger.info(f"Starting tshark: {' '.join(tshark_cmd)}") - tshark_proc = subprocess.Popen( - tshark_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - bufsize=1 - ) - register_process(tshark_proc) - logger.info(f"Started tshark (PID: {tshark_proc.pid})") - - # Check tshark didn't exit immediately - time.sleep(1) - if tshark_proc.poll() is not None: - stderr_output = '' - try: - stderr_output = tshark_proc.stderr.read() - except Exception: - pass - exit_code = tshark_proc.returncode - logger.error(f"tshark exited immediately (code: {exit_code}). stderr: {stderr_output[:500]}") - # Clean up grgsm_livemon since monitoring can't work without tshark - safe_terminate(grgsm_proc) - unregister_process(grgsm_proc) - unregister_process(tshark_proc) - raise RuntimeError(f'tshark failed (exit code {exit_code}): {stderr_output[:200]}') - - return grgsm_proc, tshark_proc, field_order - - -def _start_and_register_monitor(arfcn: int, device_index: int) -> None: - """Start monitoring processes and register them in global state. - - This is shared logic between start_monitor() and auto_start_monitor(). - Must be called within gsm_spy_lock context. - - Args: - arfcn: ARFCN to monitor - device_index: SDR device index - """ - # Start monitoring processes - grgsm_proc, tshark_proc, field_order = _start_monitoring_processes(arfcn, device_index) - app_module.gsm_spy_livemon_process = grgsm_proc - app_module.gsm_spy_monitor_process = tshark_proc - app_module.gsm_spy_selected_arfcn = arfcn - - # Start monitoring thread - monitor_thread_obj = threading.Thread( - target=monitor_thread, - args=(tshark_proc, field_order), - daemon=True - ) - monitor_thread_obj.start() - - -@gsm_spy_bp.route('/dashboard') -def dashboard(): - """Render GSM Spy dashboard.""" - return render_template( - 'gsm_spy_dashboard.html', - shared_observer_location=SHARED_OBSERVER_LOCATION_ENABLED - ) - - -@gsm_spy_bp.route('/start', methods=['POST']) -def start_scanner(): - """Start GSM scanner (G-01 BTS Scanner).""" - global gsm_towers_found, gsm_connected - - with app_module.gsm_spy_lock: - if app_module.gsm_spy_scanner_running: - return jsonify({'error': 'Scanner already running'}), 400 - - data = request.get_json() or {} - device_index = data.get('device', 0) - region = data.get('region', 'Americas') - selected_bands = data.get('bands', []) # Get user-selected bands - - # Validate device index - try: - device_index = validate_device_index(device_index) - except ValueError as e: - return jsonify({'error': str(e)}), 400 - - # Claim SDR device to prevent conflicts - from app import claim_sdr_device - claim_error = claim_sdr_device(device_index, 'GSM Spy') - if claim_error: - return jsonify({ - 'error': claim_error, - 'error_type': 'DEVICE_BUSY' - }), 409 - - # If no bands selected, use all bands for the region (backwards compatibility) - if selected_bands: - validated_bands, error = validate_band_names(selected_bands, region) - if error: - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': error}), 400 - selected_bands = validated_bands - else: - region_bands = REGIONAL_BANDS.get(region, REGIONAL_BANDS['Americas']) - selected_bands = list(region_bands.keys()) - logger.warning(f"No bands specified, using all bands for {region}: {selected_bands}") - - # Build grgsm_scanner command - # Example: grgsm_scanner --args="rtl=0" -b GSM900 - if not shutil.which('grgsm_scanner'): - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': 'grgsm_scanner not found. Please install gr-gsm.'}), 500 - - try: - cmd = ['grgsm_scanner'] - - # Add device argument (--args for RTL-SDR device selection) - cmd.extend(['--args', f'rtl={device_index}']) - - # Add selected band arguments - # Map internal band names to grgsm_scanner -b values - # grgsm_scanner accepts: GSM900, GSM850, DCS1800, PCS1900, GSM450, GSM480, GSM-R - GRGSM_BAND_MAP = { - 'EGSM900': 'GSM900', - 'EGSM900_EXT': None, # Covered by GSM900 scan - 'GSM850': 'GSM850', - 'GSM800': None, # Not a standard GSM band for grgsm_scanner - 'DCS1800': 'DCS1800', - 'PCS1900': 'PCS1900', - } - bands_added = set() - for band_name in selected_bands: - grgsm_band = GRGSM_BAND_MAP.get(band_name, band_name) - if grgsm_band is None: - logger.info(f"Skipping band {band_name} (not supported by grgsm_scanner)") - continue - if grgsm_band not in bands_added: - cmd.extend(['-b', grgsm_band]) - bands_added.add(grgsm_band) - - if not bands_added: - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': f'No scannable bands selected. ' - f'GSM800 and EGSM900_EXT are not supported by grgsm_scanner.'}), 400 - - logger.info(f"Starting GSM scanner: {' '.join(cmd)}") - - # Set a flag to indicate scanner should run - app_module.gsm_spy_active_device = device_index - app_module.gsm_spy_region = region - app_module.gsm_spy_scanner_running = True # Use as flag initially - - # Reset counters for new session - gsm_towers_found = 0 - gsm_devices_tracked = 0 - - # Start geocoding worker (if not already running) - start_geocoding_worker() - - # Start scanning thread (will run grgsm_scanner in a loop) - scanner_thread_obj = threading.Thread( - target=scanner_thread, - args=(cmd, device_index), - daemon=True - ) - scanner_thread_obj.start() - - gsm_connected = True - - return jsonify({ - 'status': 'started', - 'device': device_index, - 'region': region - }) - - except FileNotFoundError: - from app import release_sdr_device - release_sdr_device(device_index) - return jsonify({'error': 'grgsm_scanner not found. Please install gr-gsm.'}), 500 - except Exception as e: - from app import release_sdr_device - release_sdr_device(device_index) - logger.error(f"Error starting GSM scanner: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/monitor', methods=['POST']) -def start_monitor(): - """Start monitoring specific tower (G-02 Decoding).""" - with app_module.gsm_spy_lock: - if app_module.gsm_spy_monitor_process: - return jsonify({'error': 'Monitor already running'}), 400 - - data = request.get_json() or {} - arfcn = data.get('arfcn') - device_index = data.get('device', app_module.gsm_spy_active_device or 0) - - if not arfcn: - return jsonify({'error': 'ARFCN required'}), 400 - - # Validate ARFCN is valid integer and in known GSM band ranges - try: - arfcn = int(arfcn) - # This will raise ValueError if ARFCN is not in any known band - arfcn_to_frequency(arfcn) - except (ValueError, TypeError) as e: - return jsonify({'error': f'Invalid ARFCN: {e}'}), 400 - - # Validate device index - try: - device_index = validate_device_index(device_index) - except ValueError as e: - return jsonify({'error': str(e)}), 400 - - try: - # Start and register monitoring (shared logic) - _start_and_register_monitor(arfcn, device_index) - - return jsonify({ - 'status': 'monitoring', - 'arfcn': arfcn, - 'device': device_index - }) - - except FileNotFoundError as e: - return jsonify({'error': f'Tool not found: {e}'}), 500 - except Exception as e: - logger.error(f"Error starting monitor: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/stop', methods=['POST']) -def stop_scanner(): - """Stop GSM scanner and monitor.""" - global gsm_connected, gsm_towers_found, gsm_devices_tracked - - with app_module.gsm_spy_lock: - killed = [] - - # Stop scanner (now just a flag, thread will see it and exit) - if app_module.gsm_spy_scanner_running: - app_module.gsm_spy_scanner_running = False - killed.append('scanner') - - # Terminate livemon process - if app_module.gsm_spy_livemon_process: - unregister_process(app_module.gsm_spy_livemon_process) - if safe_terminate(app_module.gsm_spy_livemon_process, timeout=5): - killed.append('livemon') - app_module.gsm_spy_livemon_process = None - - # Terminate monitor process - if app_module.gsm_spy_monitor_process: - unregister_process(app_module.gsm_spy_monitor_process) - if safe_terminate(app_module.gsm_spy_monitor_process, timeout=5): - killed.append('monitor') - app_module.gsm_spy_monitor_process = None - - # Release SDR device from registry - if app_module.gsm_spy_active_device is not None: - from app import release_sdr_device - release_sdr_device(app_module.gsm_spy_active_device) - app_module.gsm_spy_active_device = None - app_module.gsm_spy_selected_arfcn = None - gsm_connected = False - gsm_towers_found = 0 - gsm_devices_tracked = 0 - - return jsonify({'status': 'stopped', 'killed': killed}) - - -@gsm_spy_bp.route('/stream') -def stream(): - """SSE stream for real-time GSM updates.""" - def generate(): - """Generate SSE events.""" - logger.info("SSE stream connected - client subscribed") - - # Send current state on connect (handles reconnects and late-joining clients) - existing_towers = dict(app_module.gsm_spy_towers.items()) - logger.info(f"SSE sending {len(existing_towers)} existing towers on connect") - for key, tower_data in existing_towers.items(): - yield format_sse(tower_data) - - last_keepalive = time.time() - idle_count = 0 # Track consecutive idle checks to handle transitions - - while True: - try: - # Check if scanner/monitor are still running - # Use idle counter to avoid disconnecting during scanner→monitor transition - if not app_module.gsm_spy_scanner_running and not app_module.gsm_spy_monitor_process: - idle_count += 1 - if idle_count >= 5: # 5 seconds grace period for mode transitions - logger.info("SSE stream: no active scanner or monitor, disconnecting") - yield format_sse({'type': 'disconnected'}) - break - else: - idle_count = 0 - - # Try to get data from queue - try: - data = app_module.gsm_spy_queue.get(timeout=1) - logger.info(f"SSE sending: type={data.get('type', '?')} keys={list(data.keys())}") - yield format_sse(data) - last_keepalive = time.time() - except queue.Empty: - # Send keepalive if needed - if time.time() - last_keepalive > 30: - yield format_sse({'type': 'keepalive'}) - last_keepalive = time.time() - - except GeneratorExit: - logger.info("SSE stream: client disconnected (GeneratorExit)") - break - except Exception as e: - logger.error(f"Error in GSM stream: {e}") - yield format_sse({'type': 'error', 'message': str(e)}) - break - - response = Response(generate(), mimetype='text/event-stream') - response.headers['Cache-Control'] = 'no-cache' - response.headers['X-Accel-Buffering'] = 'no' - response.headers['Connection'] = 'keep-alive' - return response - - -@gsm_spy_bp.route('/status') -def status(): - """Get current GSM Spy status.""" - api_usage = get_api_usage_today() - return jsonify({ - 'running': bool(app_module.gsm_spy_scanner_running), - 'monitoring': app_module.gsm_spy_monitor_process is not None, - 'towers_found': gsm_towers_found, - 'devices_tracked': gsm_devices_tracked, - 'device': app_module.gsm_spy_active_device, - 'region': app_module.gsm_spy_region, - 'selected_arfcn': app_module.gsm_spy_selected_arfcn, - 'api_usage_today': api_usage, - 'api_limit': config.GSM_API_DAILY_LIMIT, - 'api_remaining': config.GSM_API_DAILY_LIMIT - api_usage, - 'api_key_configured': bool(get_opencellid_api_key()) - }) - - -@gsm_spy_bp.route('/settings/api_key', methods=['GET', 'POST']) -def settings_api_key(): - """Get or set OpenCellID API key configuration.""" - from utils.database import get_setting, set_setting - - if request.method == 'GET': - env_key = config.GSM_OPENCELLID_API_KEY - db_key = get_setting('gsm.opencellid.api_key', '') - - if env_key: - source = 'env' - configured = True - elif db_key: - source = 'database' - configured = True - else: - source = 'none' - configured = False - - usage_today = get_api_usage_today() - - return jsonify({ - 'configured': configured, - 'source': source, - 'usage_today': usage_today, - 'api_limit': config.GSM_API_DAILY_LIMIT - }) - - # POST: save key to database - data = request.get_json() or {} - key = data.get('key', '').strip() - - if not key: - return jsonify({'error': 'API key cannot be empty'}), 400 - - set_setting('gsm.opencellid.api_key', key) - logger.info("OpenCellID API key saved to database") - - return jsonify({ - 'status': 'saved', - 'configured': True, - 'source': 'database' - }) - - -@gsm_spy_bp.route('/lookup_cell', methods=['POST']) -def lookup_cell(): - """Lookup cell tower via OpenCellID (G-05).""" - data = request.get_json() or {} - mcc = data.get('mcc') - mnc = data.get('mnc') - lac = data.get('lac') - cid = data.get('cid') - - if any(v is None for v in [mcc, mnc, lac, cid]): - return jsonify({'error': 'MCC, MNC, LAC, and CID required'}), 400 - - try: - # Check local cache first - with get_db() as conn: - result = conn.execute(''' - SELECT lat, lon, azimuth, range_meters, operator, radio - FROM gsm_cells - WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? - ''', (mcc, mnc, lac, cid)).fetchone() - - if result: - return jsonify({ - 'source': 'cache', - 'lat': result['lat'], - 'lon': result['lon'], - 'azimuth': result['azimuth'], - 'range': result['range_meters'], - 'operator': result['operator'], - 'radio': result['radio'] - }) - - # Check API key and usage limit - api_key = get_opencellid_api_key() - if not api_key: - return jsonify({'error': 'OpenCellID API key not configured'}), 503 - if not can_use_api(): - current_usage = get_api_usage_today() - return jsonify({ - 'error': 'OpenCellID API daily limit reached', - 'usage_today': current_usage, - 'limit': config.GSM_API_DAILY_LIMIT - }), 429 - - # Call OpenCellID API - api_url = config.GSM_OPENCELLID_API_URL - params = { - 'key': api_key, - 'mcc': mcc, - 'mnc': mnc, - 'lac': lac, - 'cellid': cid, - 'format': 'json' - } - - response = requests.get(api_url, params=params, timeout=10) - - if response.status_code == 200: - cell_data = response.json() - - # Increment API usage counter - usage_count = increment_api_usage() - logger.info(f"OpenCellID API call #{usage_count} today") - - # Cache the result - conn.execute(''' - INSERT OR REPLACE INTO gsm_cells - (mcc, mnc, lac, cid, lat, lon, azimuth, range_meters, samples, radio, operator, last_verified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) - ''', ( - mcc, mnc, lac, cid, - cell_data.get('lat'), - cell_data.get('lon'), - cell_data.get('azimuth'), - cell_data.get('range'), - cell_data.get('samples'), - cell_data.get('radio'), - cell_data.get('operator') - )) - conn.commit() - - return jsonify({ - 'source': 'api', - 'lat': cell_data.get('lat'), - 'lon': cell_data.get('lon'), - 'azimuth': cell_data.get('azimuth'), - 'range': cell_data.get('range'), - 'operator': cell_data.get('operator'), - 'radio': cell_data.get('radio') - }) - else: - return jsonify({'error': 'Cell not found in OpenCellID'}), 404 - - except Exception as e: - logger.error(f"Error looking up cell: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/detect_rogue', methods=['POST']) -def detect_rogue(): - """Analyze and flag rogue towers (G-07).""" - data = request.get_json() or {} - tower_info = data.get('tower') - - if not tower_info: - return jsonify({'error': 'Tower info required'}), 400 - - try: - is_rogue = False - reasons = [] - - # Check if tower exists in OpenCellID - mcc = tower_info.get('mcc') - mnc = tower_info.get('mnc') - lac = tower_info.get('lac') - cid = tower_info.get('cid') - - if all(v is not None for v in [mcc, mnc, lac, cid]): - with get_db() as conn: - result = conn.execute(''' - SELECT id FROM gsm_cells - WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? - ''', (mcc, mnc, lac, cid)).fetchone() - - if not result: - is_rogue = True - reasons.append('Tower not found in OpenCellID database') - - # Check signal strength anomalies - signal = tower_info.get('signal_strength', 0) - if signal > -50: # Suspiciously strong signal - is_rogue = True - reasons.append(f'Unusually strong signal: {signal} dBm') - - # If rogue, insert into database - if is_rogue: - with get_db() as conn: - conn.execute(''' - INSERT INTO gsm_rogues - (arfcn, mcc, mnc, lac, cid, signal_strength, reason, threat_level) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ''', ( - tower_info.get('arfcn'), - mcc, mnc, lac, cid, - signal, - '; '.join(reasons), - 'high' if len(reasons) > 1 else 'medium' - )) - conn.commit() - - return jsonify({ - 'is_rogue': is_rogue, - 'reasons': reasons - }) - - except Exception as e: - logger.error(f"Error detecting rogue: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/towers') -def get_towers(): - """Get all detected towers.""" - towers = [] - for key, tower_data in app_module.gsm_spy_towers.items(): - towers.append(tower_data) - return jsonify(towers) - - -@gsm_spy_bp.route('/devices') -def get_devices(): - """Get all tracked devices (IMSI/TMSI).""" - devices = [] - for key, device_data in app_module.gsm_spy_devices.items(): - devices.append(device_data) - return jsonify(devices) - - -@gsm_spy_bp.route('/rogues') -def get_rogues(): - """Get all detected rogue towers.""" - try: - with get_db() as conn: - results = conn.execute(''' - SELECT * FROM gsm_rogues - WHERE acknowledged = 0 - ORDER BY detected_at DESC - LIMIT 50 - ''').fetchall() - - rogues = [dict(row) for row in results] - return jsonify(rogues) - except Exception as e: - logger.error(f"Error fetching rogues: {e}") - return jsonify({'error': str(e)}), 500 - - -# ============================================ -# Advanced Features (G-08 through G-12) -# ============================================ - -@gsm_spy_bp.route('/velocity', methods=['GET']) -def get_velocity_data(): - """Get velocity vectoring data for tracked devices (G-08).""" - try: - device_id = request.args.get('device_id') - minutes = int(request.args.get('minutes', 60)) # Last 60 minutes by default - - with get_db() as conn: - # Get velocity log entries - query = ''' - SELECT * FROM gsm_velocity_log - WHERE timestamp >= datetime('now', '-' || ? || ' minutes') - ''' - params = [minutes] - - if device_id: - query += ' AND device_id = ?' - params.append(device_id) - - query += ' ORDER BY timestamp DESC LIMIT 100' - - results = conn.execute(query, params).fetchall() - velocity_data = [dict(row) for row in results] - - return jsonify(velocity_data) - except Exception as e: - logger.error(f"Error fetching velocity data: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/velocity/calculate', methods=['POST']) -def calculate_velocity(): - """Calculate velocity for a device based on TA transitions (G-08).""" - data = request.get_json() or {} - device_id = data.get('device_id') - - if not device_id: - return jsonify({'error': 'device_id required'}), 400 - - try: - with get_db() as conn: - # Get last two TA readings for this device - results = conn.execute(''' - SELECT ta_value, cid, timestamp - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - ORDER BY timestamp DESC - LIMIT 2 - ''', (device_id, device_id)).fetchall() - - if len(results) < 2: - return jsonify({'velocity': 0, 'message': 'Insufficient data'}) - - curr = dict(results[0]) - prev = dict(results[1]) - - # Calculate distance change (TA * 554 meters) - curr_distance = curr['ta_value'] * config.GSM_TA_METERS_PER_UNIT - prev_distance = prev['ta_value'] * config.GSM_TA_METERS_PER_UNIT - distance_change = abs(curr_distance - prev_distance) - - # Calculate time difference - curr_time = datetime.fromisoformat(curr['timestamp']) - prev_time = datetime.fromisoformat(prev['timestamp']) - time_diff_seconds = (curr_time - prev_time).total_seconds() - - # Calculate velocity (m/s) - if time_diff_seconds > 0: - velocity = distance_change / time_diff_seconds - else: - velocity = 0 - - # Store in velocity log - conn.execute(''' - INSERT INTO gsm_velocity_log - (device_id, prev_ta, curr_ta, prev_cid, curr_cid, estimated_velocity) - VALUES (?, ?, ?, ?, ?, ?) - ''', (device_id, prev['ta_value'], curr['ta_value'], - prev['cid'], curr['cid'], velocity)) - conn.commit() - - return jsonify({ - 'device_id': device_id, - 'velocity_mps': round(velocity, 2), - 'velocity_kmh': round(velocity * 3.6, 2), - 'distance_change_m': round(distance_change, 2), - 'time_diff_s': round(time_diff_seconds, 2) - }) - - except Exception as e: - logger.error(f"Error calculating velocity: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/crowd_density', methods=['GET']) -def get_crowd_density(): - """Get crowd density data by sector (G-09).""" - try: - hours = int(request.args.get('hours', 1)) # Last 1 hour by default - cid = request.args.get('cid') # Optional: specific cell - - with get_db() as conn: - # Count unique TMSI per cell in time window - query = ''' - SELECT - cid, - lac, - COUNT(DISTINCT tmsi) as unique_devices, - COUNT(*) as total_pings, - MIN(timestamp) as first_seen, - MAX(timestamp) as last_seen - FROM gsm_tmsi_log - WHERE timestamp >= datetime('now', '-' || ? || ' hours') - ''' - params = [hours] - - if cid: - query += ' AND cid = ?' - params.append(cid) - - query += ' GROUP BY cid, lac ORDER BY unique_devices DESC' - - results = conn.execute(query, params).fetchall() - density_data = [] - - for row in results: - density_data.append({ - 'cid': row['cid'], - 'lac': row['lac'], - 'unique_devices': row['unique_devices'], - 'total_pings': row['total_pings'], - 'first_seen': row['first_seen'], - 'last_seen': row['last_seen'], - 'density_level': 'high' if row['unique_devices'] > 20 else - 'medium' if row['unique_devices'] > 10 else 'low' - }) - - return jsonify(density_data) - - except Exception as e: - logger.error(f"Error fetching crowd density: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/life_patterns', methods=['GET']) -def get_life_patterns(): - """Get life pattern analysis for a device (G-10).""" - try: - device_id = request.args.get('device_id') - if not device_id: - # Return empty results gracefully when no device selected - return jsonify({ - 'device_id': None, - 'patterns': [], - 'message': 'No device selected' - }), 200 - - with get_db() as conn: - # Get historical signal data - results = conn.execute(''' - SELECT - strftime('%H', timestamp) as hour, - strftime('%w', timestamp) as day_of_week, - cid, - lac, - COUNT(*) as occurrences - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - AND timestamp >= datetime('now', '-60 days') - GROUP BY hour, day_of_week, cid, lac - ORDER BY occurrences DESC - ''', (device_id, device_id)).fetchall() - - patterns = [] - for row in results: - patterns.append({ - 'hour': int(row['hour']), - 'day_of_week': int(row['day_of_week']), - 'cid': row['cid'], - 'lac': row['lac'], - 'occurrences': row['occurrences'], - 'day_name': ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'][int(row['day_of_week'])] - }) - - # Identify regular patterns - regular_locations = [] - for pattern in patterns[:5]: # Top 5 most frequent - if pattern['occurrences'] >= 3: # Seen at least 3 times - regular_locations.append({ - 'cid': pattern['cid'], - 'typical_time': f"{pattern['day_name']} {pattern['hour']:02d}:00", - 'frequency': pattern['occurrences'] - }) - - return jsonify({ - 'device_id': device_id, - 'patterns': patterns, - 'regular_locations': regular_locations, - 'total_observations': sum(p['occurrences'] for p in patterns) - }) - - except Exception as e: - logger.error(f"Error analyzing life patterns: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/neighbor_audit', methods=['GET']) -def neighbor_audit(): - """Audit neighbor cell lists for consistency (G-11).""" - try: - cid = request.args.get('cid') - if not cid: - # Return empty results gracefully when no tower selected - return jsonify({ - 'cid': None, - 'neighbors': [], - 'inconsistencies': [], - 'message': 'No tower selected' - }), 200 - - with get_db() as conn: - # Get tower info with metadata (neighbor list stored in metadata JSON) - result = conn.execute(''' - SELECT metadata FROM gsm_cells WHERE cid = ? - ''', (cid,)).fetchone() - - if not result or not result['metadata']: - return jsonify({ - 'cid': cid, - 'status': 'no_data', - 'message': 'No neighbor list data available' - }) - - # Parse metadata JSON - metadata = json.loads(result['metadata']) - neighbor_list = metadata.get('neighbors', []) - - # Audit consistency - issues = [] - for neighbor_cid in neighbor_list: - # Check if neighbor exists in database - neighbor_exists = conn.execute(''' - SELECT id FROM gsm_cells WHERE cid = ? - ''', (neighbor_cid,)).fetchone() - - if not neighbor_exists: - issues.append({ - 'type': 'missing_neighbor', - 'cid': neighbor_cid, - 'message': f'Neighbor CID {neighbor_cid} not found in database' - }) - - return jsonify({ - 'cid': cid, - 'neighbor_count': len(neighbor_list), - 'neighbors': neighbor_list, - 'issues': issues, - 'status': 'suspicious' if issues else 'normal' - }) - - except Exception as e: - logger.error(f"Error auditing neighbors: {e}") - return jsonify({'error': str(e)}), 500 - - -@gsm_spy_bp.route('/traffic_correlation', methods=['GET']) -def traffic_correlation(): - """Correlate uplink/downlink traffic for pairing analysis (G-12).""" - try: - cid = request.args.get('cid') - minutes = int(request.args.get('minutes', 5)) - - with get_db() as conn: - # Get recent signal activity for this cell - results = conn.execute(''' - SELECT - imsi, - tmsi, - ta_value, - timestamp, - metadata - FROM gsm_signals - WHERE cid = ? - AND timestamp >= datetime('now', '-' || ? || ' minutes') - ORDER BY timestamp DESC - ''', (cid, minutes)).fetchall() - - correlations = [] - seen_devices = set() - - for row in results: - device_id = row['imsi'] or row['tmsi'] - if device_id and device_id not in seen_devices: - seen_devices.add(device_id) - - # Simple correlation: count bursts - burst_count = conn.execute(''' - SELECT COUNT(*) as bursts - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - AND cid = ? - AND timestamp >= datetime('now', '-' || ? || ' minutes') - ''', (device_id, device_id, cid, minutes)).fetchone() - - correlations.append({ - 'device_id': device_id, - 'burst_count': burst_count['bursts'], - 'last_seen': row['timestamp'], - 'ta_value': row['ta_value'], - 'activity_level': 'high' if burst_count['bursts'] > 10 else - 'medium' if burst_count['bursts'] > 5 else 'low' - }) - - return jsonify({ - 'cid': cid, - 'time_window_minutes': minutes, - 'active_devices': len(correlations), - 'correlations': correlations - }) - - except Exception as e: - logger.error(f"Error correlating traffic: {e}") - return jsonify({'error': str(e)}), 500 - - -# ============================================ -# Helper Functions -# ============================================ - -def parse_grgsm_scanner_output(line: str) -> dict[str, Any] | None: - """Parse grgsm_scanner output line. - - Actual output format (comma-separated key-value pairs): - ARFCN: 975, Freq: 925.2M, CID: 13522, LAC: 38722, MCC: 262, MNC: 1, Pwr: -58 - """ - try: - line = line.strip() - - # Skip non-data lines (progress, config, neighbour info, blank) - if not line or 'ARFCN:' not in line: - return None - - # Parse "ARFCN: 975, Freq: 925.2M, CID: 13522, LAC: 38722, MCC: 262, MNC: 1, Pwr: -58" - fields = {} - for part in line.split(','): - part = part.strip() - if ':' in part: - key, _, value = part.partition(':') - fields[key.strip()] = value.strip() - - if 'ARFCN' in fields and 'CID' in fields: - cid = int(fields.get('CID', 0)) - mcc = int(fields.get('MCC', 0)) - mnc = int(fields.get('MNC', 0)) - - # Only skip entries with no network identity at all (MCC=0 AND MNC=0) - # CID=0 with valid MCC/MNC is a partially decoded cell - still useful - if mcc == 0 and mnc == 0: - logger.debug(f"Skipping unidentified ARFCN (MCC=0, MNC=0): {line}") - return None - - # Freq may have 'M' suffix (e.g. "925.2M") - freq_str = fields.get('Freq', '0').rstrip('Mm') - - data = { - 'type': 'tower', - 'arfcn': int(fields['ARFCN']), - 'frequency': float(freq_str), - 'cid': cid, - 'lac': int(fields.get('LAC', 0)), - 'mcc': mcc, - 'mnc': mnc, - 'signal_strength': float(fields.get('Pwr', -999)), - 'timestamp': datetime.now().isoformat() - } - return data - - except Exception as e: - logger.debug(f"Failed to parse scanner line: {line} - {e}") - - return None - - -def parse_tshark_output(line: str, field_order: list[str] | None = None) -> dict[str, Any] | None: - """Parse tshark filtered GSM output. - - Args: - line: Tab-separated tshark output line - field_order: List of logical field names in column order. - If None, assumes legacy order: ['ta', 'tmsi', 'imsi', 'lac', 'cid'] - """ - if field_order is None: - field_order = ['ta', 'tmsi', 'imsi', 'lac', 'cid'] - - try: - parts = line.rstrip('\n\r').split('\t') - - if len(parts) < len(field_order): - return None - - # Map logical names to column values - field_map = {} - for i, logical_name in enumerate(field_order): - field_map[logical_name] = parts[i] if parts[i] else None - - # Convert types (use base 0 to auto-detect hex 0x prefix from tshark) - ta_raw = field_map.get('ta') - data = { - 'type': 'device', - 'ta_value': int(ta_raw, 0) if ta_raw else None, - 'tmsi': field_map.get('tmsi'), - 'imsi': field_map.get('imsi'), - 'lac': int(field_map['lac'], 0) if field_map.get('lac') else None, - 'cid': int(field_map['cid'], 0) if field_map.get('cid') else None, - 'timestamp': datetime.now().isoformat() - } - - # Need at least one identifier - if not data['tmsi'] and not data['imsi']: - return None - - # Calculate distance from TA - if data['ta_value'] is not None: - data['distance_meters'] = data['ta_value'] * config.GSM_TA_METERS_PER_UNIT - - return data - - except Exception as e: - logger.debug(f"Failed to parse tshark line: {line} - {e}") - - return None - - -def auto_start_monitor(tower_data): - """Automatically start monitoring the strongest tower found.""" - try: - arfcn = tower_data.get('arfcn') - if not arfcn: - logger.warning("Cannot auto-monitor: no ARFCN in tower data") - return - - logger.info(f"Auto-monitoring strongest tower: ARFCN {arfcn}, Signal {tower_data.get('signal_strength')} dBm") - - # Brief delay to ensure scanner has stabilized - time.sleep(2) - - with app_module.gsm_spy_lock: - if app_module.gsm_spy_monitor_process: - logger.info("Monitor already running, skipping auto-start") - return - - device_index = app_module.gsm_spy_active_device or 0 - - # Start and register monitoring (shared logic) - _start_and_register_monitor(arfcn, device_index) - - # Send SSE notification - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'auto_monitor_started', - 'arfcn': arfcn, - 'tower': tower_data - }) - except queue.Full: - pass - - logger.info(f"Auto-monitoring started for ARFCN {arfcn}") - - except Exception as e: - logger.error(f"Error in auto-monitoring: {e}") - - -def scanner_thread(cmd, device_index): - """Thread to continuously run grgsm_scanner in a loop with non-blocking I/O. - - grgsm_scanner scans once and exits, so we loop it to provide - continuous updates to the dashboard. - """ - global gsm_towers_found - - strongest_tower = None - auto_monitor_triggered = False # Moved outside loop - persists across scans - scan_count = 0 - crash_count = 0 - process = None - - try: - while app_module.gsm_spy_scanner_running: # Flag check - scan_count += 1 - logger.info(f"Starting GSM scan #{scan_count}") - - try: - # Start scanner process - # Set OSMO_FSM_DUP_CHECK_DISABLED to prevent libosmocore - # abort on duplicate FSM registration (common with apt gr-gsm) - env = dict(os.environ, - OSMO_FSM_DUP_CHECK_DISABLED='1', - PYTHONUNBUFFERED='1', - QT_QPA_PLATFORM='offscreen') - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - bufsize=1, - env=env - ) - register_process(process) - logger.info(f"Started grgsm_scanner (PID: {process.pid})") - - # Standard pattern: reader threads with queue - output_queue_local = queue.Queue() - - def read_stdout(): - try: - for line in iter(process.stdout.readline, ''): - if line: - output_queue_local.put(('stdout', line)) - except Exception as e: - logger.error(f"stdout read error: {e}") - finally: - output_queue_local.put(('eof', None)) - - def read_stderr(): - try: - for line in iter(process.stderr.readline, ''): - if line: - logger.debug(f"grgsm_scanner stderr: {line.strip()}") - # grgsm_scanner outputs scan results to stderr - output_queue_local.put(('stderr', line)) - except Exception as e: - logger.error(f"stderr read error: {e}") - - stdout_thread = threading.Thread(target=read_stdout, daemon=True) - stderr_thread = threading.Thread(target=read_stderr, daemon=True) - stdout_thread.start() - stderr_thread.start() - - # Process output with timeout - scan_start = time.time() - last_output = scan_start - scan_timeout = 300 # 5 minute maximum per scan (4 bands takes ~2-3 min) - - while app_module.gsm_spy_scanner_running: - # Check if process died - if process.poll() is not None: - logger.info(f"Scanner exited (code: {process.returncode})") - break - - # Get output from queue with timeout - try: - msg_type, line = output_queue_local.get(timeout=1.0) - - if msg_type == 'eof': - break # EOF - - last_output = time.time() - stripped = line.strip() - logger.info(f"Scanner [{msg_type}]: {stripped}") - - # Forward progress and status info to frontend - progress_match = re.match(r'Scanning:\s+([\d.]+)%\s+done', stripped) - if progress_match: - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'progress', - 'percent': float(progress_match.group(1)), - 'scan': scan_count - }) - except queue.Full: - pass - continue - if stripped.startswith('Try scan CCCH'): - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'status', - 'message': stripped, - 'scan': scan_count - }) - except queue.Full: - pass - - parsed = parse_grgsm_scanner_output(line) - if parsed: - # Enrich with coordinates - from utils.gsm_geocoding import enrich_tower_data - enriched = enrich_tower_data(parsed) - - # Store in DataStore - key = f"{enriched['mcc']}_{enriched['mnc']}_{enriched['lac']}_{enriched['cid']}" - app_module.gsm_spy_towers[key] = enriched - - # Track strongest tower - signal = enriched.get('signal_strength', -999) - if strongest_tower is None or signal > strongest_tower.get('signal_strength', -999): - strongest_tower = enriched - - # Queue for SSE - try: - app_module.gsm_spy_queue.put_nowait(enriched) - except queue.Full: - logger.warning("Queue full, dropping tower update") - - # Thread-safe counter update - with app_module.gsm_spy_lock: - gsm_towers_found += 1 - except queue.Empty: - # No output, check timeout - if time.time() - last_output > scan_timeout: - logger.warning(f"Scan timeout after {scan_timeout}s") - break - - # Drain remaining queue items after process exits - while not output_queue_local.empty(): - try: - msg_type, line = output_queue_local.get_nowait() - if line: - logger.info(f"Scanner [{msg_type}] (drain): {line.strip()}") - except queue.Empty: - break - - # Clean up process with timeout - if process.poll() is None: - logger.info("Terminating scanner process") - safe_terminate(process, timeout=5) - else: - process.wait() # Reap zombie - - exit_code = process.returncode - scan_duration = time.time() - scan_start - logger.info(f"Scan #{scan_count} complete (exit code: {exit_code}, duration: {scan_duration:.1f}s)") - - # Notify frontend scan completed - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'scan_complete', - 'scan': scan_count, - 'duration': round(scan_duration, 1), - 'towers_found': gsm_towers_found - }) - except queue.Full: - pass - - # Detect crash pattern: process exits too quickly with no data - if scan_duration < 5 and exit_code != 0: - crash_count += 1 - logger.error( - f"grgsm_scanner crashed on startup (exit code: {exit_code}). " - f"Crash count: {crash_count}. Check gr-gsm/libosmocore compatibility." - ) - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'error', - 'message': f'grgsm_scanner crashed (exit code: {exit_code}). ' - 'This may be a gr-gsm/libosmocore compatibility issue. ' - 'Try rebuilding gr-gsm from source.', - 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S') - }) - except queue.Full: - pass - if crash_count >= 3: - logger.error("grgsm_scanner crashed 3 times, stopping scanner") - break - - except FileNotFoundError: - logger.error( - "grgsm_scanner not found. Please install gr-gsm: " - "https://github.com/bkerler/gr-gsm" - ) - # Send error to SSE stream so the UI knows - try: - app_module.gsm_spy_queue.put({ - 'type': 'error', - 'message': 'grgsm_scanner not found. Please install gr-gsm.', - 'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S') - }) - except Exception: - pass - break # Don't retry - binary won't appear - - except Exception as e: - logger.error(f"Scanner scan error: {e}", exc_info=True) - if process and process.poll() is None: - safe_terminate(process) - - # Check if should continue - if not app_module.gsm_spy_scanner_running: - break - - # After first scan completes: auto-switch to monitoring if towers found - # Scanner process has exited so SDR is free for grgsm_livemon - if not auto_monitor_triggered and strongest_tower and scan_count >= 1: - auto_monitor_triggered = True - arfcn = strongest_tower.get('arfcn') - signal = strongest_tower.get('signal_strength', -999) - logger.info( - f"Scan complete with towers found. Auto-switching to monitor mode " - f"on ARFCN {arfcn} (signal: {signal} dBm)" - ) - - # Stop scanner loop - SDR needed for monitoring - app_module.gsm_spy_scanner_running = False - - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'status', - 'message': f'Switching to monitor mode on ARFCN {arfcn}...' - }) - except queue.Full: - pass - - # Start monitoring (SDR is free since scanner process exited) - try: - with app_module.gsm_spy_lock: - if not app_module.gsm_spy_monitor_process: - _start_and_register_monitor(arfcn, device_index) - logger.info(f"Auto-monitoring started for ARFCN {arfcn}") - - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'auto_monitor_started', - 'arfcn': arfcn, - 'tower': strongest_tower - }) - except queue.Full: - pass - except Exception as e: - logger.error(f"Error starting auto-monitor: {e}", exc_info=True) - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'error', - 'message': f'Monitor failed: {e}' - }) - except queue.Full: - pass - # Resume scanning if monitor failed - app_module.gsm_spy_scanner_running = True - - break # Exit scanner loop (monitoring takes over) - - # Wait between scans with responsive flag checking - logger.info("Waiting 5 seconds before next scan") - for i in range(5): - if not app_module.gsm_spy_scanner_running: - break - time.sleep(1) - - except Exception as e: - logger.error(f"Scanner thread fatal error: {e}", exc_info=True) - - finally: - # Always cleanup - if process and process.poll() is None: - safe_terminate(process, timeout=5) - - logger.info("Scanner thread terminated") - - # Reset global state - but don't release SDR if monitoring took over - with app_module.gsm_spy_lock: - app_module.gsm_spy_scanner_running = False - if app_module.gsm_spy_monitor_process is None: - # No monitor running - release SDR device - if app_module.gsm_spy_active_device is not None: - from app import release_sdr_device - release_sdr_device(app_module.gsm_spy_active_device) - app_module.gsm_spy_active_device = None - else: - logger.info("Monitor is running, keeping SDR device allocated") - - -def monitor_thread(process, field_order=None): - """Thread to read tshark output using standard iter pattern.""" - global gsm_devices_tracked - - # Standard pattern: reader thread with queue - output_queue_local = queue.Queue() - - def read_stdout(): - try: - for line in iter(process.stdout.readline, ''): - if line: - output_queue_local.put(('stdout', line)) - except Exception as e: - logger.error(f"tshark read error: {e}") - finally: - output_queue_local.put(('eof', None)) - - def read_stderr(): - try: - for line in iter(process.stderr.readline, ''): - if line: - logger.debug(f"tshark stderr: {line.strip()}") - except Exception: - pass - - stdout_thread = threading.Thread(target=read_stdout, daemon=True) - stdout_thread.start() - stderr_thread = threading.Thread(target=read_stderr, daemon=True) - stderr_thread.start() - - monitor_start_time = time.time() - packets_captured = 0 - lines_received = 0 - last_heartbeat = time.time() - - try: - while app_module.gsm_spy_monitor_process: - # Check if process died - if process.poll() is not None: - logger.info(f"Monitor process exited (code: {process.returncode})") - break - - # Send periodic heartbeat so frontend knows monitor is alive - now = time.time() - if now - last_heartbeat >= 5: - last_heartbeat = now - elapsed = int(now - monitor_start_time) - try: - app_module.gsm_spy_queue.put_nowait({ - 'type': 'monitor_heartbeat', - 'elapsed': elapsed, - 'packets': packets_captured, - 'devices': len(app_module.gsm_spy_devices) - }) - except queue.Full: - pass - # Periodic diagnostic: how many raw lines vs parsed - if lines_received > 0 or elapsed % 30 == 0: - logger.info( - f"Monitor stats: {lines_received} tshark lines received, " - f"{packets_captured} parsed, fields={field_order}" - ) - - # Get output from queue with timeout - try: - msg_type, line = output_queue_local.get(timeout=1.0) - except queue.Empty: - continue # Timeout, check flag again - - if msg_type == 'eof': - break # EOF - - lines_received += 1 - # Log first 5 raw lines and then every 100th for diagnostics - if lines_received <= 5 or lines_received % 100 == 0: - logger.debug(f"tshark raw line #{lines_received}: {line.rstrip()!r}") - - parsed = parse_tshark_output(line, field_order) - if parsed: - packets_captured += 1 - - # Store in DataStore - key = parsed.get('tmsi') or parsed.get('imsi') or str(time.time()) - app_module.gsm_spy_devices[key] = parsed - - # Queue for SSE stream - try: - app_module.gsm_spy_queue.put_nowait(parsed) - except queue.Full: - pass - - # Store in database for historical analysis - try: - with get_db() as conn: - # gsm_signals table - conn.execute(''' - INSERT INTO gsm_signals - (imsi, tmsi, lac, cid, ta_value, arfcn) - VALUES (?, ?, ?, ?, ?, ?) - ''', ( - parsed.get('imsi'), - parsed.get('tmsi'), - parsed.get('lac'), - parsed.get('cid'), - parsed.get('ta_value'), - app_module.gsm_spy_selected_arfcn - )) - - # gsm_tmsi_log table for crowd density - if parsed.get('tmsi'): - conn.execute(''' - INSERT INTO gsm_tmsi_log - (tmsi, lac, cid, ta_value) - VALUES (?, ?, ?, ?) - ''', ( - parsed.get('tmsi'), - parsed.get('lac'), - parsed.get('cid'), - parsed.get('ta_value') - )) - - # Velocity calculation (G-08) - device_id = parsed.get('imsi') or parsed.get('tmsi') - if device_id and parsed.get('ta_value') is not None: - # Get previous TA reading - prev_reading = conn.execute(''' - SELECT ta_value, cid, timestamp - FROM gsm_signals - WHERE (imsi = ? OR tmsi = ?) - ORDER BY timestamp DESC - LIMIT 1 OFFSET 1 - ''', (device_id, device_id)).fetchone() - - if prev_reading: - # Calculate velocity - curr_ta = parsed.get('ta_value') - prev_ta = prev_reading['ta_value'] - curr_distance = curr_ta * config.GSM_TA_METERS_PER_UNIT - prev_distance = prev_ta * config.GSM_TA_METERS_PER_UNIT - distance_change = abs(curr_distance - prev_distance) - - # Time difference - prev_time = datetime.fromisoformat(prev_reading['timestamp']) - curr_time = datetime.now() - time_diff_seconds = (curr_time - prev_time).total_seconds() - - if time_diff_seconds > 0: - velocity = distance_change / time_diff_seconds - - # Store velocity - conn.execute(''' - INSERT INTO gsm_velocity_log - (device_id, prev_ta, curr_ta, prev_cid, curr_cid, estimated_velocity) - VALUES (?, ?, ?, ?, ?, ?) - ''', ( - device_id, - prev_ta, - curr_ta, - prev_reading['cid'], - parsed.get('cid'), - velocity - )) - - conn.commit() - except Exception as e: - logger.error(f"Error storing device data: {e}") - - # Thread-safe counter - with app_module.gsm_spy_lock: - gsm_devices_tracked += 1 - - except Exception as e: - logger.error(f"Monitor thread error: {e}", exc_info=True) - - finally: - # Reap process with timeout - try: - if process.poll() is None: - process.terminate() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - logger.warning("Monitor process didn't terminate, killing") - process.kill() - process.wait() - else: - process.wait() - logger.info(f"Monitor process exited with code {process.returncode}") - except Exception as e: - logger.error(f"Error reaping monitor process: {e}") - - logger.info("Monitor thread terminated") diff --git a/setup.sh b/setup.sh index 3e0f210..f91e2b0 100755 --- a/setup.sh +++ b/setup.sh @@ -243,12 +243,6 @@ check_tools() { check_required "hcitool" "Bluetooth scan utility" hcitool check_required "hciconfig" "Bluetooth adapter config" hciconfig - echo - info "GSM Intelligence:" - check_recommended "grgsm_scanner" "GSM tower scanner (gr-gsm)" grgsm_scanner - check_recommended "grgsm_livemon" "GSM live monitor (gr-gsm)" grgsm_livemon - check_recommended "tshark" "Packet analysis (Wireshark)" tshark - echo info "SoapySDR:" check_required "SoapySDRUtil" "SoapySDR CLI utility" SoapySDRUtil @@ -713,47 +707,6 @@ install_macos_packages() { progress "Installing gpsd" brew_install gpsd - # gr-gsm for GSM Intelligence - progress "Installing gr-gsm" - if ! cmd_exists grgsm_scanner; then - brew_install gnuradio - (brew_install gr-gsm) || { - warn "gr-gsm not available in Homebrew, building from source..." - ( - tmp_dir="$(mktemp -d)" - trap 'rm -rf "$tmp_dir"' EXIT - - info "Cloning gr-gsm repository..." - git clone --depth 1 https://github.com/bkerler/gr-gsm.git "$tmp_dir/gr-gsm" >/dev/null 2>&1 \ - || { warn "Failed to clone gr-gsm. GSM Spy feature will not work."; exit 1; } - - cd "$tmp_dir/gr-gsm" - mkdir -p build && cd build - info "Compiling gr-gsm (this may take several minutes)..." - if cmake .. >/dev/null 2>&1 && make -j$(sysctl -n hw.ncpu) >/dev/null 2>&1; then - if [[ -w /usr/local/lib ]]; then - make install >/dev/null 2>&1 - else - sudo make install >/dev/null 2>&1 - fi - ok "gr-gsm installed successfully from source" - else - warn "Failed to build gr-gsm. GSM Spy feature will not work." - fi - ) - } - else - ok "gr-gsm already installed" - fi - - # Wireshark (tshark) for GSM packet analysis - progress "Installing tshark" - if ! cmd_exists tshark; then - brew_install wireshark - else - ok "tshark already installed" - fi - progress "Installing Ubertooth tools (optional)" if ! cmd_exists ubertooth-btle; then echo @@ -1164,82 +1117,6 @@ install_debian_packages() { progress "Installing gpsd" apt_install gpsd gpsd-clients || true - # gr-gsm for GSM Intelligence - progress "Installing GNU Radio and gr-gsm" - if ! cmd_exists grgsm_scanner; then - # Try to install gr-gsm directly from package repositories - apt_install gnuradio gnuradio-dev gr-osmosdr gr-gsm || { - warn "gr-gsm package not available in repositories. Attempting source build..." - - # Fallback: Build from source - progress "Building gr-gsm from source" - apt_install git cmake libboost-all-dev libcppunit-dev swig \ - doxygen liblog4cpp5-dev python3-scipy python3-numpy \ - libvolk-dev libuhd-dev libfftw3-dev || true - - info "Cloning gr-gsm repository..." - if [ -d /tmp/gr-gsm ]; then - rm -rf /tmp/gr-gsm - fi - - git clone https://github.com/bkerler/gr-gsm.git /tmp/gr-gsm || { - warn "Failed to clone gr-gsm repository. GSM Spy will not be available." - return 0 - } - - cd /tmp/gr-gsm - mkdir -p build && cd build - - # Try to find GNU Radio cmake files - if [ -d /usr/lib/x86_64-linux-gnu/cmake/gnuradio ]; then - export CMAKE_PREFIX_PATH="/usr/lib/x86_64-linux-gnu/cmake/gnuradio:$CMAKE_PREFIX_PATH" - fi - - info "Running CMake configuration..." - if cmake .. 2>/dev/null; then - info "Compiling gr-gsm (this may take several minutes)..." - if make -j$(nproc) 2>/dev/null; then - $SUDO make install - $SUDO ldconfig - cd ~ - rm -rf /tmp/gr-gsm - ok "gr-gsm built and installed successfully" - else - warn "gr-gsm compilation failed. GSM Spy feature will not work." - cd ~ - rm -rf /tmp/gr-gsm - fi - else - warn "gr-gsm CMake configuration failed. GNU Radio 3.8+ may not be available." - cd ~ - rm -rf /tmp/gr-gsm - fi - } - - # Verify installation - if cmd_exists grgsm_scanner; then - ok "gr-gsm installed successfully" - else - warn "gr-gsm installation incomplete. GSM Spy feature will not work." - fi - else - ok "gr-gsm already installed" - fi - - # Wireshark (tshark) for GSM packet analysis - progress "Installing tshark" - if ! cmd_exists tshark; then - # Pre-accept non-root capture prompt for non-interactive install - echo 'wireshark-common wireshark-common/install-setuid boolean true' | $SUDO debconf-set-selections - apt_install tshark || true - # Allow non-root capture - $SUDO dpkg-reconfigure wireshark-common 2>/dev/null || true - $SUDO usermod -a -G wireshark $USER 2>/dev/null || true - ok "tshark installed. You may need to re-login for wireshark group permissions." - else - ok "tshark already installed" - fi - progress "Installing Python packages" apt_install python3-venv python3-pip || true # Install Python packages via apt (more reliable than pip on modern Debian/Ubuntu) @@ -1327,7 +1204,7 @@ final_summary_and_hard_fail() { warn "Missing RECOMMENDED tools (some features will not work):" for t in "${missing_recommended[@]}"; do echo " - $t"; done echo - warn "Install these for full functionality (GSM Intelligence, etc.)" + warn "Install these for full functionality" fi } @@ -1376,7 +1253,7 @@ main() { install_python_deps - # Download leaflet-heat plugin for GSM heatmap (offline mode) + # Download leaflet-heat plugin (offline mode) if [ ! -f "static/vendor/leaflet-heat/leaflet-heat.js" ]; then info "Downloading leaflet-heat plugin..." mkdir -p static/vendor/leaflet-heat diff --git a/static/js/core/settings-manager.js b/static/js/core/settings-manager.js index 69ed89b..35649c2 100644 --- a/static/js/core/settings-manager.js +++ b/static/js/core/settings-manager.js @@ -946,32 +946,9 @@ function loadApiKeyStatus() { if (!badge) return; - fetch('/gsm_spy/settings/api_key') - .then(r => r.json()) - .then(data => { - if (data.configured) { - badge.textContent = 'Configured'; - badge.className = 'asset-badge available'; - desc.textContent = 'Source: ' + (data.source === 'env' ? 'Environment variable' : 'Database'); - } else { - badge.textContent = 'Not configured'; - badge.className = 'asset-badge missing'; - desc.textContent = 'No API key set'; - } - if (usage) { - usage.textContent = (data.usage_today || 0) + ' / ' + (data.api_limit || 1000); - } - if (bar) { - const pct = Math.min(100, ((data.usage_today || 0) / (data.api_limit || 1000)) * 100); - bar.style.width = pct + '%'; - bar.style.background = pct > 90 ? 'var(--accent-red)' : pct > 70 ? 'var(--accent-yellow)' : 'var(--accent-cyan)'; - } - }) - .catch(() => { - badge.textContent = 'Error'; - badge.className = 'asset-badge missing'; - desc.textContent = 'Could not load status'; - }); + badge.textContent = 'Not available'; + badge.className = 'asset-badge missing'; + desc.textContent = 'GSM feature removed'; } /** @@ -994,30 +971,8 @@ function saveApiKey() { result.style.color = 'var(--text-dim)'; result.textContent = 'Saving...'; - fetch('/gsm_spy/settings/api_key', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ key: key }) - }) - .then(r => r.json()) - .then(data => { - if (data.error) { - result.style.color = 'var(--accent-red)'; - result.textContent = data.error; - } else { - result.style.color = 'var(--accent-green)'; - result.textContent = 'API key saved successfully.'; - input.value = ''; - loadApiKeyStatus(); - // Hide the banner if visible - const banner = document.getElementById('apiKeyBanner'); - if (banner) banner.style.display = 'none'; - } - }) - .catch(() => { - result.style.color = 'var(--accent-red)'; - result.textContent = 'Error saving API key.'; - }); + result.style.color = 'var(--accent-red)'; + result.textContent = 'GSM feature has been removed.'; } /** diff --git a/templates/gsm_spy_dashboard.html b/templates/gsm_spy_dashboard.html deleted file mode 100644 index 85b12a2..0000000 --- a/templates/gsm_spy_dashboard.html +++ /dev/null @@ -1,3300 +0,0 @@ - - - - - - GSM SPY // INTERCEPT - See the Invisible - - {% if offline_settings.fonts_source == 'local' %} - - {% else %} - - {% endif %} - - {% if offline_settings.assets_source == 'local' %} - - - {% else %} - - - {% endif %} - - {% if offline_settings.assets_source == 'local' %} - - {% else %} - - {% endif %} - - - - - - - - - - - -
-
- -
- -
-
- STANDBY -
-
- - {% set active_mode = 'gsm' %} - {% include 'partials/nav.html' with context %} - - - - - {% include 'partials/settings-modal.html' %} - - -
-
-
- 0 - TOWERS -
-
- 0 - DEVICES -
-
- 0 - ROGUES -
-
- 0 - SIGNALS -
-
- - - CROWD -
-
-
-
- STANDBY -
-
--:--:-- UTC
- - -
-
- - -
-
-
-
Analytics Overview
- -
-
-
- -
-
-
📍
-
Velocity Tracking
-
-
- Track device movement by analyzing Timing Advance transitions and cell handovers. - Estimates velocity and direction based on TA delta and cell sector patterns. -
-
-
-
0
-
Devices Tracked
-
-
-
- km/h
-
Avg Velocity
-
-
-
- - -
-
-
👥
-
Crowd Density
-
-
- Aggregate TMSI pings per cell sector to estimate crowd density. - Visualizes hotspots and congestion patterns across towers. -
-
-
-
0
-
Total Devices
-
-
-
0
-
Peak Sector
-
-
-
- - -
-
-
📊
-
Life Patterns
-
-
- Analyze 60-day historical data to identify recurring patterns in device behavior. - Detects work locations, commute routes, and daily routines. -
-
-
-
0
-
Patterns Found
-
-
-
0%
-
Confidence
-
-
-
- - -
-
-
🔍
-
Neighbor Audit
-
-
- Validate neighbor cell lists against expected network topology. - Detects inconsistencies that may indicate rogue towers. -
-
-
-
0
-
Neighbors
-
-
-
0
-
Anomalies
-
-
-
- - -
-
-
📡
-
Traffic Correlation
-
-
- Correlate uplink and downlink timing to identify communication patterns. - Maps device-to-device interactions and network flows. -
-
-
-
0
-
Paired Flows
-
-
-
0
-
Active Now
-
-
-
-
-
-
-
- -
- -
- -
-
- SIGNAL SOURCE -
-
-
-
- - -
-
-
- - -
-
- REGION / BANDS -
-
-
- - - -
-
- - -
-
- ADVANCED ANALYSIS -
-
-
- - - - - -
- -
-
- - -
- - -
-
- - -
- -
-
- SELECTED TOWER -
-
-
-
-
📡
-
Select a tower from the list
-
-
-
- - -
-
- TRACKED TOWERS -
-
-
-
-
No towers detected
-
Start scanner to begin
-
-
-
- - -
-
- TRACKED DEVICES -
-
-
-
-
No devices tracked
-
Devices will appear here
-
-
-
- - -
-
- ALERTS -
-
-
-
-
No alerts
-
-
-
-
- - -
- -
- GPS LOCATION -
- - - -
-
- - -
- GSM SCANNER -
- - - -
-
-
-
- - -
-
-
-
- Device Detail - -
-
-
-
-
- - - - - - - - - diff --git a/templates/index.html b/templates/index.html index 373005e..7d635ef 100644 --- a/templates/index.html +++ b/templates/index.html @@ -171,10 +171,6 @@ Vessels - - - GSM SPY - -
@@ -360,70 +359,6 @@ - -
-
-
OpenCellID API Key
-

- Required for GSM cell tower geolocation. Get a free key at - opencellid.org/register - (1,000 lookups/day). -

- -
-
- Status - Checking... -
- Checking... -
- -
-
- API Key - Paste your OpenCellID API token -
-
- - -
-
- -
- -
- - -
- -
-
Usage Today
-
-
- API Calls - -- / -- -
-
-
-
-
-
- -
- Note: The environment variable INTERCEPT_GSM_OPENCELLID_API_KEY takes priority over the saved key. - Keys saved here persist across restarts. -
-
-
diff --git a/tests/test_gsm_spy.py b/tests/test_gsm_spy.py deleted file mode 100644 index 797f794..0000000 --- a/tests/test_gsm_spy.py +++ /dev/null @@ -1,360 +0,0 @@ -"""Unit tests for GSM Spy parsing and validation functions.""" - -import pytest -from routes.gsm_spy import ( - parse_grgsm_scanner_output, - parse_tshark_output, - arfcn_to_frequency, - validate_band_names, - REGIONAL_BANDS -) - - -class TestParseGrgsmScannerOutput: - """Tests for parse_grgsm_scanner_output().""" - - def test_valid_output_line(self): - """Test parsing a valid grgsm_scanner output line.""" - line = "ARFCN: 23, Freq: 940.6M, CID: 31245, LAC: 1234, MCC: 214, MNC: 01, Pwr: -48" - result = parse_grgsm_scanner_output(line) - - assert result is not None - assert result['type'] == 'tower' - assert result['arfcn'] == 23 - assert result['frequency'] == 940.6 - assert result['cid'] == 31245 - assert result['lac'] == 1234 - assert result['mcc'] == 214 - assert result['mnc'] == 1 - assert result['signal_strength'] == -48.0 - assert 'timestamp' in result - - def test_freq_without_suffix(self): - """Test parsing frequency without M suffix.""" - line = "ARFCN: 975, Freq: 925.2, CID: 13522, LAC: 38722, MCC: 262, MNC: 1, Pwr: -58" - result = parse_grgsm_scanner_output(line) - assert result is not None - assert result['frequency'] == 925.2 - - def test_config_line(self): - """Test that configuration lines are skipped.""" - line = " Configuration: 1 CCCH, not combined" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_neighbour_line(self): - """Test that neighbour cell lines are skipped.""" - line = " Neighbour Cells: 57, 61, 70, 71, 72, 86" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_cell_arfcn_line(self): - """Test that cell ARFCN lines are skipped.""" - line = " Cell ARFCNs: 63, 76" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_progress_line(self): - """Test that progress/status lines are skipped.""" - line = "Scanning GSM900 band..." - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_empty_line(self): - """Test handling of empty lines.""" - result = parse_grgsm_scanner_output("") - assert result is None - - def test_invalid_data(self): - """Test handling of non-numeric values.""" - line = "ARFCN: abc, Freq: xyz, CID: bad, LAC: data, MCC: bad, MNC: bad, Pwr: bad" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_no_identity_filtered(self): - """Test that MCC=0/MNC=0 entries (no network identity) are filtered out.""" - line = "ARFCN: 115, Freq: 925.0M, CID: 0, LAC: 0, MCC: 0, MNC: 0, Pwr: -100" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_mcc_zero_mnc_zero_filtered(self): - """Test that MCC=0/MNC=0 even with valid CID is filtered out.""" - line = "ARFCN: 113, Freq: 924.6M, CID: 1234, LAC: 5678, MCC: 0, MNC: 0, Pwr: -90" - result = parse_grgsm_scanner_output(line) - assert result is None - - def test_cid_zero_valid_mcc_passes(self): - """Test that CID=0 with valid MCC/MNC passes (partially decoded cell).""" - line = "ARFCN: 115, Freq: 958.0M, CID: 0, LAC: 21864, MCC: 234, MNC: 10, Pwr: -51" - result = parse_grgsm_scanner_output(line) - assert result is not None - assert result['cid'] == 0 - assert result['mcc'] == 234 - assert result['signal_strength'] == -51.0 - - def test_valid_cid_nonzero(self): - """Test that valid non-zero CID/MCC entries pass through.""" - line = "ARFCN: 115, Freq: 925.0M, CID: 19088, LAC: 21864, MCC: 234, MNC: 10, Pwr: -58" - result = parse_grgsm_scanner_output(line) - assert result is not None - assert result['cid'] == 19088 - assert result['signal_strength'] == -58.0 - - -class TestParseTsharkOutput: - """Tests for parse_tshark_output().""" - - def test_valid_full_output(self): - """Test parsing tshark output with all fields.""" - line = "5\t0xABCD1234\t123456789012345\t1234\t31245" - result = parse_tshark_output(line) - - assert result is not None - assert result['type'] == 'device' - assert result['ta_value'] == 5 - assert result['tmsi'] == '0xABCD1234' - assert result['imsi'] == '123456789012345' - assert result['lac'] == 1234 - assert result['cid'] == 31245 - assert result['distance_meters'] == 5 * 554 # TA * 554 meters - assert 'timestamp' in result - - def test_missing_optional_fields(self): - """Test parsing with missing optional fields (empty tabs). - - A packet with TA but no TMSI/IMSI is discarded since there's - no device identifier to track. - """ - line = "3\t\t\t1234\t31245" - result = parse_tshark_output(line) - assert result is None - - def test_missing_optional_fields_with_tmsi(self): - """Test parsing with TMSI but missing TA, IMSI, CID.""" - line = "\t0xABCD\t\t1234\t" - result = parse_tshark_output(line) - - assert result is not None - assert result['ta_value'] is None - assert result['tmsi'] == '0xABCD' - assert result['imsi'] is None - assert result['lac'] == 1234 - assert result['cid'] is None - - def test_no_ta_value(self): - """Test parsing without TA value (empty first field).""" - line = "\t0xABCD1234\t123456789012345\t1234\t31245" - result = parse_tshark_output(line) - - assert result is not None - assert result['ta_value'] is None - assert result['tmsi'] == '0xABCD1234' - assert result['imsi'] == '123456789012345' - assert result['lac'] == 1234 - assert result['cid'] == 31245 - - def test_invalid_line(self): - """Test handling of invalid tshark output.""" - line = "invalid data" - result = parse_tshark_output(line) - assert result is None - - def test_empty_line(self): - """Test handling of empty lines.""" - result = parse_tshark_output("") - assert result is None - - def test_partial_fields(self): - """Test with fewer than 5 fields.""" - line = "5\t0xABCD1234" # Only 2 fields - result = parse_tshark_output(line) - assert result is None - - -class TestArfcnToFrequency: - """Tests for arfcn_to_frequency().""" - - def test_gsm850_arfcn(self): - """Test ARFCN in GSM850 band.""" - # GSM850: ARFCN 128-251, 869-894 MHz - arfcn = 128 - freq = arfcn_to_frequency(arfcn) - assert freq == 869000000 # 869 MHz - - arfcn = 251 - freq = arfcn_to_frequency(arfcn) - assert freq == 893600000 # 893.6 MHz - - def test_egsm900_arfcn(self): - """Test ARFCN in EGSM900 band.""" - # EGSM900: ARFCN 0-124, DL = 935 + 0.2*ARFCN MHz - arfcn = 0 - freq = arfcn_to_frequency(arfcn) - assert freq == 935000000 # 935.0 MHz - - arfcn = 22 - freq = arfcn_to_frequency(arfcn) - assert freq == 939400000 # 939.4 MHz - - arfcn = 124 - freq = arfcn_to_frequency(arfcn) - assert freq == 959800000 # 959.8 MHz - - def test_egsm900_ext_arfcn(self): - """Test ARFCN in EGSM900 extension band.""" - # EGSM900_EXT: ARFCN 975-1023, DL = 925.2 + 0.2*(ARFCN-975) MHz - arfcn = 975 - freq = arfcn_to_frequency(arfcn) - assert freq == 925200000 # 925.2 MHz - - arfcn = 1023 - freq = arfcn_to_frequency(arfcn) - assert freq == 934800000 # 934.8 MHz - - def test_dcs1800_arfcn(self): - """Test ARFCN in DCS1800 band.""" - # DCS1800: ARFCN 512-885, 1805-1880 MHz - # Note: ARFCN 512 also exists in PCS1900 and will match that first - # Use ARFCN 811+ which is only in DCS1800 - arfcn = 811 # Beyond PCS1900 range (512-810) - freq = arfcn_to_frequency(arfcn) - # 811 is ARFCN offset from 512, so freq = 1805MHz + (811-512)*200kHz - expected = 1805000000 + (811 - 512) * 200000 - assert freq == expected - - arfcn = 885 - freq = arfcn_to_frequency(arfcn) - assert freq == 1879600000 # 1879.6 MHz - - def test_pcs1900_arfcn(self): - """Test ARFCN in PCS1900 band.""" - # PCS1900: ARFCN 512-810, 1930-1990 MHz - # Note: overlaps with DCS1800 ARFCN range, but different frequencies - arfcn = 512 - freq = arfcn_to_frequency(arfcn) - # Will match first band (DCS1800 in Europe config) - assert freq > 0 - - def test_invalid_arfcn(self): - """Test ARFCN outside known ranges.""" - with pytest.raises(ValueError, match="not found in any known GSM band"): - arfcn_to_frequency(9999) - - with pytest.raises(ValueError): - arfcn_to_frequency(-1) - - def test_arfcn_200khz_spacing(self): - """Test that ARFCNs are 200kHz apart.""" - arfcn1 = 128 - arfcn2 = 129 - freq1 = arfcn_to_frequency(arfcn1) - freq2 = arfcn_to_frequency(arfcn2) - assert freq2 - freq1 == 200000 # 200 kHz - - -class TestValidateBandNames: - """Tests for validate_band_names().""" - - def test_valid_americas_bands(self): - """Test valid band names for Americas region.""" - bands = ['GSM850', 'PCS1900'] - result, error = validate_band_names(bands, 'Americas') - assert result == bands - assert error is None - - def test_valid_europe_bands(self): - """Test valid band names for Europe region.""" - # Note: Europe uses EGSM900, not GSM900 - bands = ['EGSM900', 'DCS1800', 'GSM850', 'GSM800'] - result, error = validate_band_names(bands, 'Europe') - assert result == bands - assert error is None - - def test_valid_asia_bands(self): - """Test valid band names for Asia region.""" - # Note: Asia uses EGSM900, not GSM900 - bands = ['EGSM900', 'DCS1800'] - result, error = validate_band_names(bands, 'Asia') - assert result == bands - assert error is None - - def test_invalid_band_for_region(self): - """Test invalid band name for a region.""" - bands = ['GSM900', 'INVALID_BAND'] - result, error = validate_band_names(bands, 'Americas') - assert result == [] - assert error is not None - assert 'Invalid bands' in error - assert 'INVALID_BAND' in error - - def test_invalid_region(self): - """Test invalid region name.""" - bands = ['GSM900'] - result, error = validate_band_names(bands, 'InvalidRegion') - assert result == [] - assert error is not None - assert 'Invalid region' in error - - def test_empty_bands_list(self): - """Test with empty bands list.""" - result, error = validate_band_names([], 'Americas') - assert result == [] - assert error is None - - def test_single_valid_band(self): - """Test with single valid band.""" - bands = ['GSM850'] - result, error = validate_band_names(bands, 'Americas') - assert result == ['GSM850'] - assert error is None - - def test_case_sensitive_band_names(self): - """Test that band names are case-sensitive.""" - bands = ['gsm850'] # lowercase - result, error = validate_band_names(bands, 'Americas') - assert result == [] - assert error is not None - - def test_multiple_invalid_bands(self): - """Test with multiple invalid bands.""" - bands = ['INVALID1', 'GSM850', 'INVALID2'] - result, error = validate_band_names(bands, 'Americas') - assert result == [] - assert error is not None - assert 'INVALID1' in error - assert 'INVALID2' in error - - -class TestRegionalBandsConfig: - """Tests for REGIONAL_BANDS configuration.""" - - def test_all_regions_defined(self): - """Test that all expected regions are defined.""" - assert 'Americas' in REGIONAL_BANDS - assert 'Europe' in REGIONAL_BANDS - assert 'Asia' in REGIONAL_BANDS - - def test_all_bands_have_required_fields(self): - """Test that all bands have required configuration fields.""" - for region, bands in REGIONAL_BANDS.items(): - for band_name, band_config in bands.items(): - assert 'start' in band_config - assert 'end' in band_config - assert 'arfcn_start' in band_config - assert 'arfcn_end' in band_config - - def test_frequency_ranges_valid(self): - """Test that frequency ranges are positive and start < end.""" - for region, bands in REGIONAL_BANDS.items(): - for band_name, band_config in bands.items(): - assert band_config['start'] > 0 - assert band_config['end'] > 0 - assert band_config['start'] < band_config['end'] - - def test_arfcn_ranges_valid(self): - """Test that ARFCN ranges are valid.""" - for region, bands in REGIONAL_BANDS.items(): - for band_name, band_config in bands.items(): - assert band_config['arfcn_start'] >= 0 - assert band_config['arfcn_end'] >= 0 - assert band_config['arfcn_start'] <= band_config['arfcn_end'] diff --git a/utils/constants.py b/utils/constants.py index f51124e..2b5edff 100644 --- a/utils/constants.py +++ b/utils/constants.py @@ -275,13 +275,3 @@ MAX_DEAUTH_ALERTS_AGE_SECONDS = 300 # 5 minutes # Deauth detector sniff timeout (seconds) DEAUTH_SNIFF_TIMEOUT = 0.5 - -# ============================================================================= -# GSM SPY (Cellular Intelligence) -# ============================================================================= - -# Maximum age for GSM tower/device data in DataStore (seconds) -MAX_GSM_AGE_SECONDS = 300 # 5 minutes - -# Timing Advance conversion to meters -GSM_TA_METERS_PER_UNIT = 554 diff --git a/utils/database.py b/utils/database.py index 9e62d87..3211624 100644 --- a/utils/database.py +++ b/utils/database.py @@ -453,134 +453,6 @@ def init_db() -> None: ON tscm_cases(status, created_at) ''') - # ===================================================================== - # GSM (Global System for Mobile) Intelligence Tables - # ===================================================================== - - # gsm_cells - Known cell towers (OpenCellID cache) - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_cells ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - mcc INTEGER NOT NULL, - mnc INTEGER NOT NULL, - lac INTEGER NOT NULL, - cid INTEGER NOT NULL, - lat REAL, - lon REAL, - azimuth INTEGER, - range_meters INTEGER, - samples INTEGER, - radio TEXT, - operator TEXT, - first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - last_verified TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - metadata TEXT, - UNIQUE(mcc, mnc, lac, cid) - ) - ''') - - # gsm_rogues - Detected rogue towers / IMSI catchers - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_rogues ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - arfcn INTEGER NOT NULL, - mcc INTEGER, - mnc INTEGER, - lac INTEGER, - cid INTEGER, - signal_strength REAL, - reason TEXT NOT NULL, - threat_level TEXT DEFAULT 'medium', - detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - location_lat REAL, - location_lon REAL, - acknowledged BOOLEAN DEFAULT 0, - notes TEXT, - metadata TEXT - ) - ''') - - # gsm_signals - 60-day archive of signal observations - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_signals ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - imsi TEXT, - tmsi TEXT, - mcc INTEGER, - mnc INTEGER, - lac INTEGER, - cid INTEGER, - ta_value INTEGER, - signal_strength REAL, - arfcn INTEGER, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - metadata TEXT - ) - ''') - - # gsm_tmsi_log - 24-hour raw pings for crowd density - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_tmsi_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - tmsi TEXT NOT NULL, - lac INTEGER, - cid INTEGER, - ta_value INTEGER, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - ''') - - # gsm_velocity_log - 1-hour buffer for movement tracking - conn.execute(''' - CREATE TABLE IF NOT EXISTS gsm_velocity_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - device_id TEXT NOT NULL, - prev_ta INTEGER, - curr_ta INTEGER, - prev_cid INTEGER, - curr_cid INTEGER, - timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - estimated_velocity REAL, - metadata TEXT - ) - ''') - - # GSM indexes for performance - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_cells_location - ON gsm_cells(lat, lon) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_cells_identity - ON gsm_cells(mcc, mnc, lac, cid) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_rogues_severity - ON gsm_rogues(threat_level, detected_at) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_signals_cell_time - ON gsm_signals(cid, lac, timestamp) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_signals_device - ON gsm_signals(imsi, tmsi, timestamp) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_tmsi_log_time - ON gsm_tmsi_log(timestamp) - ''') - - conn.execute(''' - CREATE INDEX IF NOT EXISTS idx_gsm_velocity_log_device - ON gsm_velocity_log(device_id, timestamp) - ''') - # ===================================================================== # DSC (Digital Selective Calling) Tables # ===================================================================== @@ -2298,60 +2170,3 @@ def cleanup_old_payloads(max_age_hours: int = 24) -> int: ''', (f'-{max_age_hours} hours',)) return cursor.rowcount - -# ============================================================================= -# GSM Cleanup Functions -# ============================================================================= - -def cleanup_old_gsm_signals(max_age_days: int = 60) -> int: - """ - Remove old GSM signal observations (60-day archive). - - Args: - max_age_days: Maximum age in days (default: 60) - - Returns: - Number of deleted entries - """ - with get_db() as conn: - cursor = conn.execute(''' - DELETE FROM gsm_signals - WHERE timestamp < datetime('now', ?) - ''', (f'-{max_age_days} days',)) - return cursor.rowcount - - -def cleanup_old_gsm_tmsi_log(max_age_hours: int = 24) -> int: - """ - Remove old TMSI log entries (24-hour buffer for crowd density). - - Args: - max_age_hours: Maximum age in hours (default: 24) - - Returns: - Number of deleted entries - """ - with get_db() as conn: - cursor = conn.execute(''' - DELETE FROM gsm_tmsi_log - WHERE timestamp < datetime('now', ?) - ''', (f'-{max_age_hours} hours',)) - return cursor.rowcount - - -def cleanup_old_gsm_velocity_log(max_age_hours: int = 1) -> int: - """ - Remove old velocity log entries (1-hour buffer for movement tracking). - - Args: - max_age_hours: Maximum age in hours (default: 1) - - Returns: - Number of deleted entries - """ - with get_db() as conn: - cursor = conn.execute(''' - DELETE FROM gsm_velocity_log - WHERE timestamp < datetime('now', ?) - ''', (f'-{max_age_hours} hours',)) - return cursor.rowcount diff --git a/utils/dependencies.py b/utils/dependencies.py index e6a1bee..a12eca7 100644 --- a/utils/dependencies.py +++ b/utils/dependencies.py @@ -444,38 +444,6 @@ TOOL_DEPENDENCIES = { } } }, - 'gsm': { - 'name': 'GSM Intelligence', - 'tools': { - 'grgsm_scanner': { - 'required': True, - 'description': 'gr-gsm scanner for finding GSM towers', - 'install': { - 'apt': 'Build gr-gsm from source: https://github.com/bkerler/gr-gsm', - 'brew': 'brew install gr-gsm (may require manual build)', - 'manual': 'https://github.com/bkerler/gr-gsm' - } - }, - 'grgsm_livemon': { - 'required': True, - 'description': 'gr-gsm live monitor for decoding GSM signals', - 'install': { - 'apt': 'Included with gr-gsm package', - 'brew': 'Included with gr-gsm', - 'manual': 'Included with gr-gsm' - } - }, - 'tshark': { - 'required': True, - 'description': 'Wireshark CLI for parsing GSM packets', - 'install': { - 'apt': 'sudo apt-get install tshark', - 'brew': 'brew install wireshark', - 'manual': 'https://www.wireshark.org/download.html' - } - } - } - } } diff --git a/utils/gsm_geocoding.py b/utils/gsm_geocoding.py deleted file mode 100644 index feaf164..0000000 --- a/utils/gsm_geocoding.py +++ /dev/null @@ -1,226 +0,0 @@ -"""GSM Cell Tower Geocoding Service. - -Provides hybrid cache-first geocoding with async API fallback for cell towers. -""" - -from __future__ import annotations - -import logging -import queue -from typing import Any - -import requests - -import config -from utils.database import get_db - -logger = logging.getLogger('intercept.gsm_geocoding') - -# Queue for pending geocoding requests -_geocoding_queue = queue.Queue(maxsize=100) - - -def lookup_cell_coordinates(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None: - """ - Lookup cell tower coordinates with cache-first strategy. - - Strategy: - 1. Check gsm_cells table (cache) - fast synchronous lookup - 2. If not found, return None (caller decides whether to use API) - - Args: - mcc: Mobile Country Code - mnc: Mobile Network Code - lac: Location Area Code - cid: Cell ID - - Returns: - dict with keys: lat, lon, source='cache', azimuth (optional), - range_meters (optional), operator (optional), radio (optional) - Returns None if not found in cache. - """ - try: - with get_db() as conn: - result = conn.execute(''' - SELECT lat, lon, azimuth, range_meters, operator, radio - FROM gsm_cells - WHERE mcc = ? AND mnc = ? AND lac = ? AND cid = ? - ''', (mcc, mnc, lac, cid)).fetchone() - - if result and result['lat'] is not None and result['lon'] is not None: - return { - 'lat': result['lat'], - 'lon': result['lon'], - 'source': 'cache', - 'azimuth': result['azimuth'], - 'range_meters': result['range_meters'], - 'operator': result['operator'], - 'radio': result['radio'] - } - - return None - - except Exception as e: - logger.error(f"Error looking up coordinates from cache: {e}") - return None - - -def _get_api_key() -> str: - """Get OpenCellID API key at runtime (env var first, then database).""" - env_key = config.GSM_OPENCELLID_API_KEY - if env_key: - return env_key - from utils.database import get_setting - return get_setting('gsm.opencellid.api_key', '') - - -def lookup_cell_from_api(mcc: int, mnc: int, lac: int, cid: int) -> dict[str, Any] | None: - """ - Lookup cell tower from OpenCellID API and cache result. - - Args: - mcc: Mobile Country Code - mnc: Mobile Network Code - lac: Location Area Code - cid: Cell ID - - Returns: - dict with keys: lat, lon, source='api', azimuth (optional), - range_meters (optional), operator (optional), radio (optional) - Returns None if API call fails or cell not found. - """ - try: - api_key = _get_api_key() - if not api_key: - logger.warning("OpenCellID API key not configured") - return None - - api_url = config.GSM_OPENCELLID_API_URL - params = { - 'key': api_key, - 'mcc': mcc, - 'mnc': mnc, - 'lac': lac, - 'cellid': cid, - 'format': 'json' - } - - response = requests.get(api_url, params=params, timeout=10) - - if response.status_code == 200: - cell_data = response.json() - lat = cell_data.get('lat') - lon = cell_data.get('lon') - - # Validate response has actual coordinates - if lat is None or lon is None: - logger.warning( - f"OpenCellID API returned 200 but no coordinates for " - f"MCC={mcc} MNC={mnc} LAC={lac} CID={cid}: {cell_data}" - ) - return None - - # Cache the result - with get_db() as conn: - conn.execute(''' - INSERT OR REPLACE INTO gsm_cells - (mcc, mnc, lac, cid, lat, lon, azimuth, range_meters, samples, radio, operator, last_verified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) - ''', ( - mcc, mnc, lac, cid, - lat, lon, - cell_data.get('azimuth'), - cell_data.get('range'), - cell_data.get('samples'), - cell_data.get('radio'), - cell_data.get('operator') - )) - conn.commit() - - logger.info(f"Cached cell tower from API: MCC={mcc} MNC={mnc} LAC={lac} CID={cid} -> ({lat}, {lon})") - - return { - 'lat': lat, - 'lon': lon, - 'source': 'api', - 'azimuth': cell_data.get('azimuth'), - 'range_meters': cell_data.get('range'), - 'operator': cell_data.get('operator'), - 'radio': cell_data.get('radio') - } - else: - logger.warning( - f"OpenCellID API returned {response.status_code} for " - f"MCC={mcc} MNC={mnc} LAC={lac} CID={cid}: {response.text[:200]}" - ) - return None - - except Exception as e: - logger.error(f"Error calling OpenCellID API: {e}") - return None - - -def enrich_tower_data(tower_data: dict[str, Any]) -> dict[str, Any]: - """ - Enrich tower data with coordinates using cache-first strategy. - - If coordinates found in cache, adds them immediately. - If not found, marks as 'pending' and queues for background API lookup. - - Args: - tower_data: Dictionary with keys mcc, mnc, lac, cid (and other tower data) - - Returns: - Enriched tower_data dict with added fields: - - lat, lon (if found in cache) - - status='pending' (if needs API lookup) - - source='cache' (if from cache) - """ - mcc = tower_data.get('mcc') - mnc = tower_data.get('mnc') - lac = tower_data.get('lac') - cid = tower_data.get('cid') - - # Validate required fields - if not all([mcc is not None, mnc is not None, lac is not None, cid is not None]): - logger.warning(f"Tower data missing required fields: {tower_data}") - return tower_data - - # Try cache lookup - coords = lookup_cell_coordinates(mcc, mnc, lac, cid) - - if coords: - # Found in cache - add coordinates immediately - tower_data['lat'] = coords['lat'] - tower_data['lon'] = coords['lon'] - tower_data['source'] = 'cache' - - # Add optional fields if available - if coords.get('azimuth') is not None: - tower_data['azimuth'] = coords['azimuth'] - if coords.get('range_meters') is not None: - tower_data['range_meters'] = coords['range_meters'] - if coords.get('operator'): - tower_data['operator'] = coords['operator'] - if coords.get('radio'): - tower_data['radio'] = coords['radio'] - - logger.debug(f"Cache hit for tower: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - else: - # Not in cache - mark as pending and queue for API lookup - tower_data['status'] = 'pending' - tower_data['source'] = 'unknown' - - # Queue for background geocoding (non-blocking) - try: - _geocoding_queue.put_nowait(tower_data.copy()) - logger.debug(f"Queued tower for geocoding: MCC={mcc} MNC={mnc} LAC={lac} CID={cid}") - except queue.Full: - logger.warning("Geocoding queue full, dropping tower") - - return tower_data - - -def get_geocoding_queue() -> queue.Queue: - """Get the geocoding queue for the background worker.""" - return _geocoding_queue diff --git a/utils/logging.py b/utils/logging.py index addbabe..3d2cc6a 100644 --- a/utils/logging.py +++ b/utils/logging.py @@ -28,4 +28,3 @@ wifi_logger = get_logger('intercept.wifi') bluetooth_logger = get_logger('intercept.bluetooth') adsb_logger = get_logger('intercept.adsb') satellite_logger = get_logger('intercept.satellite') -gsm_spy_logger = get_logger('intercept.gsm_spy') From 1a4af214bf3d9cfba0686b297dc1109d33f374e4 Mon Sep 17 00:00:00 2001 From: Smittix Date: Sun, 8 Feb 2026 22:13:28 +0000 Subject: [PATCH 19/24] Fix APRS crash on large station count and station list overflow - Fix infinite loop in updateAprsStationList: querySelectorAll returns a static NodeList so the while(cards.length > 50) loop never terminated, crashing the page. Use live childElementCount instead. - Fix station list pushing map off-screen by adding overflow:hidden and min-height:0 to flex containers so only the station list scrolls. - Cap backend aprs_stations dict at 500 entries with oldest-eviction to prevent unbounded memory growth. Co-Authored-By: Claude Opus 4.6 --- routes/aprs.py | 26 +++++++++++++++++--------- templates/index.html | 22 +++++++++++----------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/routes/aprs.py b/routes/aprs.py index fbbced7..cfd346a 100644 --- a/routes/aprs.py +++ b/routes/aprs.py @@ -21,8 +21,8 @@ from flask import Blueprint, jsonify, request, Response import app as app_module from utils.logging import sensor_logger as logger from utils.validation import validate_device_index, validate_gain, validate_ppm -from utils.sse import format_sse -from utils.event_pipeline import process_event +from utils.sse import format_sse +from utils.event_pipeline import process_event from utils.constants import ( PROCESS_TERMINATE_TIMEOUT, SSE_KEEPALIVE_INTERVAL, @@ -53,6 +53,7 @@ aprs_packet_count = 0 aprs_station_count = 0 aprs_last_packet_time = None aprs_stations = {} # callsign -> station data +APRS_MAX_STATIONS = 500 # Limit tracked stations to prevent memory growth # Meter rate limiting _last_meter_time = 0.0 @@ -1371,6 +1372,13 @@ def stream_aprs_output(rtl_process: subprocess.Popen, decoder_process: subproces 'last_seen': packet.get('timestamp'), 'packet_type': packet.get('packet_type'), } + # Evict oldest stations when limit is exceeded + if len(aprs_stations) > APRS_MAX_STATIONS: + oldest = min( + aprs_stations, + key=lambda k: aprs_stations[k].get('last_seen', ''), + ) + del aprs_stations[oldest] app_module.aprs_queue.put(packet) @@ -1726,13 +1734,13 @@ def stream_aprs() -> Response: while True: try: - msg = app_module.aprs_queue.get(timeout=SSE_QUEUE_TIMEOUT) - last_keepalive = time.time() - try: - process_event('aprs', msg, msg.get('type')) - except Exception: - pass - yield format_sse(msg) + msg = app_module.aprs_queue.get(timeout=SSE_QUEUE_TIMEOUT) + last_keepalive = time.time() + try: + process_event('aprs', msg, msg.get('type')) + except Exception: + pass + yield format_sse(msg) except queue.Empty: now = time.time() if now - last_keepalive >= SSE_KEEPALIVE_INTERVAL: diff --git a/templates/index.html b/templates/index.html index 7d635ef..b40c8e3 100644 --- a/templates/index.html +++ b/templates/index.html @@ -905,7 +905,7 @@
-