Mirror/intercept
1
0
Fork 0
mirror of https://github.com/smittix/intercept.git synced 2026-06-09 06:31:55 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge upstream main: add DMR, WebSDR, HF SSTV, alerts, recordings, waterfall

Browse Source 
Merges upstream changes into fork while preserving weather satellite
(NOAA APT/Meteor LRPT via SatDump), rtlamr, multi-arch build, and
decoder console features from our branch.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Mitch Ross
2026-02-07 14:29:09 -05:00
parent 4bf35cf786 3ab1501a90
commit 99f42f66b2
88 changed files with 14535 additions and 1927 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/alerts.py
+443
View File
@@ -0,0 +1,443 @@
"""Alerting engine for cross-mode events."""
from __future__ import annotations
import json
import logging
import queue
import re
import threading
import time
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Generator
from config import ALERT_WEBHOOK_URL, ALERT_WEBHOOK_TIMEOUT, ALERT_WEBHOOK_SECRET
from utils.database import get_db
logger = logging.getLogger('intercept.alerts')
@dataclass
class AlertRule:
id: int
name: str
mode: str | None
event_type: str | None
match: dict
severity: str
enabled: bool
notify: dict
created_at: str | None = None
class AlertManager:
def __init__(self) -> None:
self._queue: queue.Queue = queue.Queue(maxsize=1000)
self._rules_cache: list[AlertRule] = []
self._rules_loaded_at = 0.0
self._cache_lock = threading.Lock()
# ------------------------------------------------------------------
# Rule management
# ------------------------------------------------------------------
def invalidate_cache(self) -> None:
with self._cache_lock:
self._rules_loaded_at = 0.0
def _load_rules(self) -> None:
with get_db() as conn:
cursor = conn.execute('''
SELECT id, name, mode, event_type, match, severity, enabled, notify, created_at
FROM alert_rules
WHERE enabled = 1
ORDER BY id ASC
''')
rules: list[AlertRule] = []
for row in cursor:
match = {}
notify = {}
try:
match = json.loads(row['match']) if row['match'] else {}
except json.JSONDecodeError:
match = {}
try:
notify = json.loads(row['notify']) if row['notify'] else {}
except json.JSONDecodeError:
notify = {}
rules.append(AlertRule(
id=row['id'],
name=row['name'],
mode=row['mode'],
event_type=row['event_type'],
match=match,
severity=row['severity'] or 'medium',
enabled=bool(row['enabled']),
notify=notify,
created_at=row['created_at'],
))
with self._cache_lock:
self._rules_cache = rules
self._rules_loaded_at = time.time()
def _get_rules(self) -> list[AlertRule]:
with self._cache_lock:
stale = (time.time() - self._rules_loaded_at) > 10
if stale:
self._load_rules()
with self._cache_lock:
return list(self._rules_cache)
def list_rules(self, include_disabled: bool = False) -> list[dict]:
with get_db() as conn:
if include_disabled:
cursor = conn.execute('''
SELECT id, name, mode, event_type, match, severity, enabled, notify, created_at
FROM alert_rules
ORDER BY id DESC
''')
else:
cursor = conn.execute('''
SELECT id, name, mode, event_type, match, severity, enabled, notify, created_at
FROM alert_rules
WHERE enabled = 1
ORDER BY id DESC
''')
return [
{
'id': row['id'],
'name': row['name'],
'mode': row['mode'],
'event_type': row['event_type'],
'match': json.loads(row['match']) if row['match'] else {},
'severity': row['severity'],
'enabled': bool(row['enabled']),
'notify': json.loads(row['notify']) if row['notify'] else {},
'created_at': row['created_at'],
}
for row in cursor
]
def add_rule(self, rule: dict) -> int:
with get_db() as conn:
cursor = conn.execute('''
INSERT INTO alert_rules (name, mode, event_type, match, severity, enabled, notify)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
rule.get('name') or 'Alert Rule',
rule.get('mode'),
rule.get('event_type'),
json.dumps(rule.get('match') or {}),
rule.get('severity') or 'medium',
1 if rule.get('enabled', True) else 0,
json.dumps(rule.get('notify') or {}),
))
rule_id = cursor.lastrowid
self.invalidate_cache()
return int(rule_id)
def update_rule(self, rule_id: int, updates: dict) -> bool:
fields = []
params = []
for key in ('name', 'mode', 'event_type', 'severity'):
if key in updates:
fields.append(f"{key} = ?")
params.append(updates[key])
if 'enabled' in updates:
fields.append('enabled = ?')
params.append(1 if updates['enabled'] else 0)
if 'match' in updates:
fields.append('match = ?')
params.append(json.dumps(updates['match'] or {}))
if 'notify' in updates:
fields.append('notify = ?')
params.append(json.dumps(updates['notify'] or {}))
if not fields:
return False
params.append(rule_id)
with get_db() as conn:
cursor = conn.execute(
f"UPDATE alert_rules SET {', '.join(fields)} WHERE id = ?",
params
)
updated = cursor.rowcount > 0
if updated:
self.invalidate_cache()
return updated
def delete_rule(self, rule_id: int) -> bool:
with get_db() as conn:
cursor = conn.execute('DELETE FROM alert_rules WHERE id = ?', (rule_id,))
deleted = cursor.rowcount > 0
if deleted:
self.invalidate_cache()
return deleted
def list_events(self, limit: int = 100, mode: str | None = None, severity: str | None = None) -> list[dict]:
query = 'SELECT id, rule_id, mode, event_type, severity, title, message, payload, created_at FROM alert_events'
clauses = []
params: list[Any] = []
if mode:
clauses.append('mode = ?')
params.append(mode)
if severity:
clauses.append('severity = ?')
params.append(severity)
if clauses:
query += ' WHERE ' + ' AND '.join(clauses)
query += ' ORDER BY id DESC LIMIT ?'
params.append(limit)
with get_db() as conn:
cursor = conn.execute(query, params)
events = []
for row in cursor:
events.append({
'id': row['id'],
'rule_id': row['rule_id'],
'mode': row['mode'],
'event_type': row['event_type'],
'severity': row['severity'],
'title': row['title'],
'message': row['message'],
'payload': json.loads(row['payload']) if row['payload'] else {},
'created_at': row['created_at'],
})
return events
# ------------------------------------------------------------------
# Event processing
# ------------------------------------------------------------------
def process_event(self, mode: str, event: dict, event_type: str | None = None) -> None:
if not isinstance(event, dict):
return
if event_type in ('keepalive', 'ping', 'status'):
return
rules = self._get_rules()
if not rules:
return
for rule in rules:
if rule.mode and rule.mode != mode:
continue
if rule.event_type and event_type and rule.event_type != event_type:
continue
if rule.event_type and not event_type:
continue
if not self._match_rule(rule.match, event):
continue
title = rule.name or 'Alert'
message = self._build_message(rule, event, event_type)
payload = {
'mode': mode,
'event_type': event_type,
'event': event,
'rule': {
'id': rule.id,
'name': rule.name,
},
}
event_id = self._store_event(rule.id, mode, event_type, rule.severity, title, message, payload)
alert_payload = {
'id': event_id,
'rule_id': rule.id,
'mode': mode,
'event_type': event_type,
'severity': rule.severity,
'title': title,
'message': message,
'payload': payload,
'created_at': datetime.now(timezone.utc).isoformat(),
}
self._queue_event(alert_payload)
self._maybe_send_webhook(alert_payload, rule.notify)
def _build_message(self, rule: AlertRule, event: dict, event_type: str | None) -> str:
if isinstance(rule.notify, dict) and rule.notify.get('message'):
return str(rule.notify.get('message'))
summary_bits = []
if event_type:
summary_bits.append(event_type)
if 'name' in event:
summary_bits.append(str(event.get('name')))
if 'ssid' in event:
summary_bits.append(str(event.get('ssid')))
if 'bssid' in event:
summary_bits.append(str(event.get('bssid')))
if 'address' in event:
summary_bits.append(str(event.get('address')))
if 'mac' in event:
summary_bits.append(str(event.get('mac')))
summary = ' | '.join(summary_bits) if summary_bits else 'Alert triggered'
return summary
def _store_event(
self,
rule_id: int,
mode: str,
event_type: str | None,
severity: str,
title: str,
message: str,
payload: dict,
) -> int:
with get_db() as conn:
cursor = conn.execute('''
INSERT INTO alert_events (rule_id, mode, event_type, severity, title, message, payload)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
rule_id,
mode,
event_type,
severity,
title,
message,
json.dumps(payload),
))
return int(cursor.lastrowid)
def _queue_event(self, alert_payload: dict) -> None:
try:
self._queue.put_nowait(alert_payload)
except queue.Full:
try:
self._queue.get_nowait()
self._queue.put_nowait(alert_payload)
except queue.Empty:
pass
def _maybe_send_webhook(self, payload: dict, notify: dict) -> None:
if not ALERT_WEBHOOK_URL:
return
if isinstance(notify, dict) and notify.get('webhook') is False:
return
try:
import urllib.request
req = urllib.request.Request(
ALERT_WEBHOOK_URL,
data=json.dumps(payload).encode('utf-8'),
headers={
'Content-Type': 'application/json',
'User-Agent': 'Intercept-Alert',
'X-Alert-Token': ALERT_WEBHOOK_SECRET or '',
},
method='POST'
)
with urllib.request.urlopen(req, timeout=ALERT_WEBHOOK_TIMEOUT) as _:
pass
except Exception as e:
logger.debug(f"Alert webhook failed: {e}")
# ------------------------------------------------------------------
# Matching
# ------------------------------------------------------------------
def _match_rule(self, rule_match: dict, event: dict) -> bool:
if not rule_match:
return True
for key, expected in rule_match.items():
actual = self._extract_value(event, key)
if not self._match_value(actual, expected):
return False
return True
def _extract_value(self, event: dict, key: str) -> Any:
if '.' not in key:
return event.get(key)
current: Any = event
for part in key.split('.'):
if isinstance(current, dict):
current = current.get(part)
else:
return None
return current
def _match_value(self, actual: Any, expected: Any) -> bool:
if isinstance(expected, dict) and 'op' in expected:
op = expected.get('op')
value = expected.get('value')
return self._apply_op(op, actual, value)
if isinstance(expected, list):
return actual in expected
if isinstance(expected, str):
if actual is None:
return False
return str(actual).lower() == expected.lower()
return actual == expected
def _apply_op(self, op: str, actual: Any, value: Any) -> bool:
if op == 'exists':
return actual is not None
if op == 'eq':
return actual == value
if op == 'neq':
return actual != value
if op == 'gt':
return _safe_number(actual) is not None and _safe_number(actual) > _safe_number(value)
if op == 'gte':
return _safe_number(actual) is not None and _safe_number(actual) >= _safe_number(value)
if op == 'lt':
return _safe_number(actual) is not None and _safe_number(actual) < _safe_number(value)
if op == 'lte':
return _safe_number(actual) is not None and _safe_number(actual) <= _safe_number(value)
if op == 'in':
return actual in (value or [])
if op == 'contains':
if actual is None:
return False
if isinstance(actual, list):
return any(str(value).lower() in str(item).lower() for item in actual)
return str(value).lower() in str(actual).lower()
if op == 'regex':
if actual is None or value is None:
return False
try:
return re.search(str(value), str(actual)) is not None
except re.error:
return False
return False
# ------------------------------------------------------------------
# Streaming
# ------------------------------------------------------------------
def stream_events(self, timeout: float = 1.0) -> Generator[dict, None, None]:
while True:
try:
event = self._queue.get(timeout=timeout)
yield event
except queue.Empty:
yield {'type': 'keepalive'}
_alert_manager: AlertManager | None = None
_alert_lock = threading.Lock()
def get_alert_manager() -> AlertManager:
global _alert_manager
with _alert_lock:
if _alert_manager is None:
_alert_manager = AlertManager()
return _alert_manager
def _safe_number(value: Any) -> float | None:
try:
return float(value)
except (TypeError, ValueError):
return None
utils/bluetooth/models.py
+13 -10
View File
@@ -148,9 +148,10 @@ class BTDeviceAggregate:
is_strong_stable: bool = False
has_random_address: bool = False
# Baseline tracking
in_baseline: bool = False
baseline_id: Optional[int] = None
# Baseline tracking
in_baseline: bool = False
baseline_id: Optional[int] = None
seen_before: bool = False
# Tracker detection fields
is_tracker: bool = False
@@ -274,9 +275,10 @@ class BTDeviceAggregate:
},
'heuristic_flags': self.heuristic_flags,
# Baseline
'in_baseline': self.in_baseline,
'baseline_id': self.baseline_id,
# Baseline
'in_baseline': self.in_baseline,
'baseline_id': self.baseline_id,
'seen_before': self.seen_before,
# Tracker detection
'tracker': {
@@ -325,10 +327,11 @@ class BTDeviceAggregate:
'last_seen': self.last_seen.isoformat(),
'age_seconds': self.age_seconds,
'seen_count': self.seen_count,
'heuristic_flags': self.heuristic_flags,
'in_baseline': self.in_baseline,
# Tracker info for list view
'is_tracker': self.is_tracker,
'heuristic_flags': self.heuristic_flags,
'in_baseline': self.in_baseline,
'seen_before': self.seen_before,
# Tracker info for list view
'is_tracker': self.is_tracker,
'tracker_type': self.tracker_type,
'tracker_name': self.tracker_name,
'tracker_confidence': self.tracker_confidence,
utils/database.py
+132 -70
View File
@@ -88,19 +88,65 @@ def init_db() -> None:
ON signal_history(mode, device_id, timestamp)
''')
# Device correlation table
conn.execute('''
CREATE TABLE IF NOT EXISTS device_correlations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
wifi_mac TEXT,
bt_mac TEXT,
confidence REAL,
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
UNIQUE(wifi_mac, bt_mac)
)
''')
# Device correlation table
conn.execute('''
CREATE TABLE IF NOT EXISTS device_correlations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
wifi_mac TEXT,
bt_mac TEXT,
confidence REAL,
first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
metadata TEXT,
UNIQUE(wifi_mac, bt_mac)
)
''')
# Alert rules
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
mode TEXT,
event_type TEXT,
match TEXT,
severity TEXT DEFAULT 'medium',
enabled BOOLEAN DEFAULT 1,
notify TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# Alert events
conn.execute('''
CREATE TABLE IF NOT EXISTS alert_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id INTEGER,
mode TEXT,
event_type TEXT,
severity TEXT DEFAULT 'medium',
title TEXT,
message TEXT,
payload TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (rule_id) REFERENCES alert_rules(id) ON DELETE SET NULL
)
''')
# Session recordings
conn.execute('''
CREATE TABLE IF NOT EXISTS recording_sessions (
id TEXT PRIMARY KEY,
mode TEXT NOT NULL,
label TEXT,
started_at TIMESTAMP NOT NULL,
stopped_at TIMESTAMP,
file_path TEXT NOT NULL,
event_count INTEGER DEFAULT 0,
size_bytes INTEGER DEFAULT 0,
metadata TEXT
)
''')
# Users table for authentication
conn.execute('''
@@ -131,20 +177,29 @@ def init_db() -> None:
# =====================================================================
# TSCM Baselines - Environment snapshots for comparison
conn.execute('''
CREATE TABLE IF NOT EXISTS tscm_baselines (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
location TEXT,
description TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
wifi_networks TEXT,
bt_devices TEXT,
rf_frequencies TEXT,
gps_coords TEXT,
is_active BOOLEAN DEFAULT 0
)
''')
conn.execute('''
CREATE TABLE IF NOT EXISTS tscm_baselines (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
location TEXT,
description TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
wifi_networks TEXT,
wifi_clients TEXT,
bt_devices TEXT,
rf_frequencies TEXT,
gps_coords TEXT,
is_active BOOLEAN DEFAULT 0
)
''')
# Ensure new columns exist for older databases
try:
columns = {row['name'] for row in conn.execute("PRAGMA table_info(tscm_baselines)")}
if 'wifi_clients' not in columns:
conn.execute('ALTER TABLE tscm_baselines ADD COLUMN wifi_clients TEXT')
except Exception as e:
logger.debug(f"Schema update skipped for tscm_baselines: {e}")
# TSCM Sweeps - Individual sweep sessions
conn.execute('''
@@ -685,15 +740,16 @@ def get_correlations(min_confidence: float = 0.5) -> list[dict]:
# TSCM Functions
# =============================================================================
def create_tscm_baseline(
name: str,
location: str | None = None,
description: str | None = None,
wifi_networks: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None,
gps_coords: dict | None = None
) -> int:
def create_tscm_baseline(
name: str,
location: str | None = None,
description: str | None = None,
wifi_networks: list | None = None,
wifi_clients: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None,
gps_coords: dict | None = None
) -> int:
"""
Create a new TSCM baseline.
@@ -701,19 +757,20 @@ def create_tscm_baseline(
The ID of the created baseline
"""
with get_db() as conn:
cursor = conn.execute('''
INSERT INTO tscm_baselines
(name, location, description, wifi_networks, bt_devices, rf_frequencies, gps_coords)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
name,
location,
description,
json.dumps(wifi_networks) if wifi_networks else None,
json.dumps(bt_devices) if bt_devices else None,
json.dumps(rf_frequencies) if rf_frequencies else None,
json.dumps(gps_coords) if gps_coords else None
))
cursor = conn.execute('''
INSERT INTO tscm_baselines
(name, location, description, wifi_networks, wifi_clients, bt_devices, rf_frequencies, gps_coords)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
name,
location,
description,
json.dumps(wifi_networks) if wifi_networks else None,
json.dumps(wifi_clients) if wifi_clients else None,
json.dumps(bt_devices) if bt_devices else None,
json.dumps(rf_frequencies) if rf_frequencies else None,
json.dumps(gps_coords) if gps_coords else None
))
return cursor.lastrowid
@@ -728,18 +785,19 @@ def get_tscm_baseline(baseline_id: int) -> dict | None:
if row is None:
return None
return {
'id': row['id'],
'name': row['name'],
'location': row['location'],
'description': row['description'],
'created_at': row['created_at'],
'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
'is_active': bool(row['is_active'])
}
return {
'id': row['id'],
'name': row['name'],
'location': row['location'],
'description': row['description'],
'created_at': row['created_at'],
'wifi_networks': json.loads(row['wifi_networks']) if row['wifi_networks'] else [],
'wifi_clients': json.loads(row['wifi_clients']) if row['wifi_clients'] else [],
'bt_devices': json.loads(row['bt_devices']) if row['bt_devices'] else [],
'rf_frequencies': json.loads(row['rf_frequencies']) if row['rf_frequencies'] else [],
'gps_coords': json.loads(row['gps_coords']) if row['gps_coords'] else None,
'is_active': bool(row['is_active'])
}
def get_all_tscm_baselines() -> list[dict]:
@@ -781,19 +839,23 @@ def set_active_tscm_baseline(baseline_id: int) -> bool:
return cursor.rowcount > 0
def update_tscm_baseline(
baseline_id: int,
wifi_networks: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None
) -> bool:
def update_tscm_baseline(
baseline_id: int,
wifi_networks: list | None = None,
wifi_clients: list | None = None,
bt_devices: list | None = None,
rf_frequencies: list | None = None
) -> bool:
"""Update baseline device lists."""
updates = []
params = []
if wifi_networks is not None:
updates.append('wifi_networks = ?')
params.append(json.dumps(wifi_networks))
if wifi_networks is not None:
updates.append('wifi_networks = ?')
params.append(json.dumps(wifi_networks))
if wifi_clients is not None:
updates.append('wifi_clients = ?')
params.append(json.dumps(wifi_clients))
if bt_devices is not None:
updates.append('bt_devices = ?')
params.append(json.dumps(bt_devices))
utils/event_pipeline.py
+29
View File
@@ -0,0 +1,29 @@
"""Shared event pipeline for alerts and recordings."""
from __future__ import annotations
from typing import Any
from utils.alerts import get_alert_manager
from utils.recording import get_recording_manager
IGNORE_TYPES = {'keepalive', 'ping'}
def process_event(mode: str, event: dict | Any, event_type: str | None = None) -> None:
if event_type in IGNORE_TYPES:
return
if not isinstance(event, dict):
return
try:
get_recording_manager().record_event(mode, event, event_type)
except Exception:
# Recording failures should never break streaming
pass
try:
get_alert_manager().process_event(mode, event, event_type)
except Exception:
# Alert failures should never break streaming
pass
utils/kiwisdr.py
+288
View File
@@ -0,0 +1,288 @@
"""KiwiSDR WebSocket audio client.
Connects to a KiwiSDR receiver via its WebSocket API and streams
decoded PCM audio back through a callback.
"""
from __future__ import annotations
import struct
import threading
import time
from typing import Optional, Callable
try:
import websocket # websocket-client library
WEBSOCKET_CLIENT_AVAILABLE = True
except ImportError:
WEBSOCKET_CLIENT_AVAILABLE = False
from utils.logging import get_logger
logger = get_logger('intercept.kiwisdr')
# Protocol constants
KIWI_KEEPALIVE_INTERVAL = 5.0
KIWI_SAMPLE_RATE = 12000 # 12 kHz mono
KIWI_SND_HEADER_SIZE = 10 # "SND"(3) + flags(1) + seq(4) + smeter(2)
KIWI_DEFAULT_PORT = 8073
VALID_MODES = ('am', 'usb', 'lsb', 'cw')
# Default bandpass filters per mode (Hz)
MODE_FILTERS = {
'am': (-4500, 4500),
'usb': (300, 3000),
'lsb': (-3000, -300),
'cw': (300, 800),
}
def parse_host_port(url: str) -> tuple[str, int]:
"""Extract host and port from a KiwiSDR URL like 'http://host:port'.
Returns (host, port) tuple. Defaults to port 8073 if not specified.
"""
if not url:
return ('', KIWI_DEFAULT_PORT)
# Strip protocol
cleaned = url
for prefix in ('http://', 'https://', 'ws://', 'wss://'):
if cleaned.lower().startswith(prefix):
cleaned = cleaned[len(prefix):]
break
# Strip path
cleaned = cleaned.split('/')[0]
# Split host:port
if ':' in cleaned:
parts = cleaned.rsplit(':', 1)
host = parts[0]
try:
port = int(parts[1])
except ValueError:
port = KIWI_DEFAULT_PORT
else:
host = cleaned
port = KIWI_DEFAULT_PORT
return (host, port)
class KiwiSDRClient:
"""Manages a WebSocket connection to a single KiwiSDR receiver."""
def __init__(
self,
host: str,
port: int = KIWI_DEFAULT_PORT,
on_audio: Optional[Callable[[bytes, int], None]] = None,
on_error: Optional[Callable[[str], None]] = None,
on_disconnect: Optional[Callable[[], None]] = None,
password: str = '',
):
self.host = host
self.port = port
self.password = password
self._on_audio = on_audio
self._on_error = on_error
self._on_disconnect = on_disconnect
self._ws = None
self._connected = False
self._stopping = False
self._receive_thread: Optional[threading.Thread] = None
self._keepalive_thread: Optional[threading.Thread] = None
self._send_lock = threading.Lock()
self.frequency_khz: float = 0
self.mode: str = 'am'
self.last_smeter: int = 0
@property
def connected(self) -> bool:
return self._connected
def connect(self, frequency_khz: float, mode: str = 'am') -> bool:
"""Connect to KiwiSDR and start receiving audio."""
if not WEBSOCKET_CLIENT_AVAILABLE:
logger.error("websocket-client not installed")
return False
if self._connected:
self.disconnect()
self.frequency_khz = frequency_khz
self.mode = mode if mode in VALID_MODES else 'am'
self._stopping = False
ws_url = self._build_ws_url()
logger.info(f"Connecting to KiwiSDR: {ws_url}")
try:
self._ws = websocket.WebSocket()
self._ws.settimeout(10)
self._ws.connect(ws_url)
# Auth
self._send('SET auth t=kiwi p=' + self.password)
time.sleep(0.2)
# Request uncompressed PCM
self._send('SET compression=0')
# Set AGC
self._send('SET agc=1 hang=0 thresh=-100 slope=6 decay=1000 manGain=50')
# Tune to frequency
self._send_tune(frequency_khz, self.mode)
# Request audio start
self._send('SET AR OK in=12000 out=44100')
self._connected = True
# Start receive thread
self._receive_thread = threading.Thread(
target=self._receive_loop, daemon=True, name='kiwi-rx'
)
self._receive_thread.start()
# Start keepalive thread
self._keepalive_thread = threading.Thread(
target=self._keepalive_loop, daemon=True, name='kiwi-ka'
)
self._keepalive_thread.start()
logger.info(f"Connected to KiwiSDR {self.host}:{self.port} @ {frequency_khz} kHz {self.mode}")
return True
except Exception as e:
logger.error(f"KiwiSDR connection failed: {e}")
self._cleanup()
return False
def tune(self, frequency_khz: float, mode: str = 'am') -> bool:
"""Retune without disconnecting."""
if not self._connected or not self._ws:
return False
self.frequency_khz = frequency_khz
if mode in VALID_MODES:
self.mode = mode
try:
self._send_tune(frequency_khz, self.mode)
logger.info(f"Retuned to {frequency_khz} kHz {self.mode}")
return True
except Exception as e:
logger.error(f"Retune failed: {e}")
return False
def disconnect(self) -> None:
"""Cleanly disconnect from KiwiSDR."""
self._stopping = True
self._connected = False
self._cleanup()
logger.info("Disconnected from KiwiSDR")
def _build_ws_url(self) -> str:
ts = int(time.time() * 1000)
return f'ws://{self.host}:{self.port}/{ts}/SND'
def _send(self, msg: str) -> None:
with self._send_lock:
if self._ws:
self._ws.send(msg)
def _send_tune(self, freq_khz: float, mode: str) -> None:
low_cut, high_cut = MODE_FILTERS.get(mode, MODE_FILTERS['am'])
self._send(f'SET mod={mode} low_cut={low_cut} high_cut={high_cut} freq={freq_khz}')
def _receive_loop(self) -> None:
"""Background thread: read frames from KiwiSDR WebSocket."""
try:
while self._connected and not self._stopping:
try:
if not self._ws:
break
self._ws.settimeout(2.0)
data = self._ws.recv()
except websocket.WebSocketTimeoutException:
continue
except Exception as e:
if not self._stopping:
logger.error(f"KiwiSDR receive error: {e}")
break
if not data or not isinstance(data, bytes):
# Text message (status/config) — ignore
continue
self._parse_snd_frame(data)
except Exception as e:
if not self._stopping:
logger.error(f"KiwiSDR receive loop error: {e}")
finally:
if not self._stopping:
self._connected = False
if self._on_disconnect:
try:
self._on_disconnect()
except Exception:
pass
def _parse_snd_frame(self, data: bytes) -> None:
"""Parse a KiwiSDR SND binary frame."""
if len(data) < KIWI_SND_HEADER_SIZE:
return
# Check header magic
if data[:3] != b'SND':
return
# flags = data[3]
# seq = struct.unpack('>I', data[4:8])[0]
# S-meter: big-endian int16 at offset 8
smeter_raw = struct.unpack('>h', data[8:10])[0]
self.last_smeter = smeter_raw
# PCM audio data starts at offset 10
pcm_data = data[KIWI_SND_HEADER_SIZE:]
if pcm_data and self._on_audio:
try:
self._on_audio(pcm_data, smeter_raw)
except Exception:
pass
def _keepalive_loop(self) -> None:
"""Background thread: send keepalive every 5 seconds."""
while self._connected and not self._stopping:
time.sleep(KIWI_KEEPALIVE_INTERVAL)
if self._connected and not self._stopping:
try:
self._send('SET keepalive')
except Exception:
break
def _cleanup(self) -> None:
"""Close WebSocket and join threads."""
if self._ws:
try:
self._ws.close()
except Exception:
pass
self._ws = None
if self._receive_thread and self._receive_thread.is_alive():
self._receive_thread.join(timeout=3.0)
if self._keepalive_thread and self._keepalive_thread.is_alive():
self._keepalive_thread.join(timeout=3.0)
self._receive_thread = None
self._keepalive_thread = None
utils/recording.py
+222
View File
@@ -0,0 +1,222 @@
"""Session recording utilities for SSE/event streams."""
from __future__ import annotations
import json
import logging
import threading
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from utils.database import get_db
logger = logging.getLogger('intercept.recording')
RECORDING_ROOT = Path(__file__).parent.parent / 'instance' / 'recordings'
@dataclass
class RecordingSession:
id: str
mode: str
label: str | None
file_path: Path
started_at: datetime
stopped_at: datetime | None = None
event_count: int = 0
size_bytes: int = 0
metadata: dict | None = None
_file_handle: Any | None = None
_lock: threading.Lock = threading.Lock()
def open(self) -> None:
self.file_path.parent.mkdir(parents=True, exist_ok=True)
self._file_handle = self.file_path.open('a', encoding='utf-8')
def close(self) -> None:
if self._file_handle:
self._file_handle.flush()
self._file_handle.close()
self._file_handle = None
def write_event(self, record: dict) -> None:
if not self._file_handle:
self.open()
line = json.dumps(record, ensure_ascii=True) + '\n'
with self._lock:
self._file_handle.write(line)
self._file_handle.flush()
self.event_count += 1
self.size_bytes += len(line.encode('utf-8'))
class RecordingManager:
def __init__(self) -> None:
self._active_by_mode: dict[str, RecordingSession] = {}
self._active_by_id: dict[str, RecordingSession] = {}
self._lock = threading.Lock()
def start_recording(self, mode: str, label: str | None = None, metadata: dict | None = None) -> RecordingSession:
with self._lock:
existing = self._active_by_mode.get(mode)
if existing:
return existing
session_id = str(uuid.uuid4())
started_at = datetime.now(timezone.utc)
filename = f"{mode}_{started_at.strftime('%Y%m%d_%H%M%S')}_{session_id}.jsonl"
file_path = RECORDING_ROOT / mode / filename
session = RecordingSession(
id=session_id,
mode=mode,
label=label,
file_path=file_path,
started_at=started_at,
metadata=metadata or {},
)
session.open()
self._active_by_mode[mode] = session
self._active_by_id[session_id] = session
with get_db() as conn:
conn.execute('''
INSERT INTO recording_sessions
(id, mode, label, started_at, file_path, event_count, size_bytes, metadata)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
session.id,
session.mode,
session.label,
session.started_at.isoformat(),
str(session.file_path),
session.event_count,
session.size_bytes,
json.dumps(session.metadata or {}),
))
return session
def stop_recording(self, mode: str | None = None, session_id: str | None = None) -> RecordingSession | None:
with self._lock:
session = None
if session_id:
session = self._active_by_id.get(session_id)
elif mode:
session = self._active_by_mode.get(mode)
if not session:
return None
session.stopped_at = datetime.now(timezone.utc)
session.close()
self._active_by_mode.pop(session.mode, None)
self._active_by_id.pop(session.id, None)
with get_db() as conn:
conn.execute('''
UPDATE recording_sessions
SET stopped_at = ?, event_count = ?, size_bytes = ?
WHERE id = ?
''', (
session.stopped_at.isoformat(),
session.event_count,
session.size_bytes,
session.id,
))
return session
def record_event(self, mode: str, event: dict, event_type: str | None = None) -> None:
if event_type in ('keepalive', 'ping'):
return
session = self._active_by_mode.get(mode)
if not session:
return
record = {
'timestamp': datetime.now(timezone.utc).isoformat(),
'mode': mode,
'event_type': event_type,
'event': event,
}
try:
session.write_event(record)
except Exception as e:
logger.debug(f"Recording write failed: {e}")
def list_recordings(self, limit: int = 50) -> list[dict]:
with get_db() as conn:
cursor = conn.execute('''
SELECT id, mode, label, started_at, stopped_at, file_path, event_count, size_bytes, metadata
FROM recording_sessions
ORDER BY started_at DESC
LIMIT ?
''', (limit,))
rows = []
for row in cursor:
rows.append({
'id': row['id'],
'mode': row['mode'],
'label': row['label'],
'started_at': row['started_at'],
'stopped_at': row['stopped_at'],
'file_path': row['file_path'],
'event_count': row['event_count'],
'size_bytes': row['size_bytes'],
'metadata': json.loads(row['metadata']) if row['metadata'] else {},
})
return rows
def get_recording(self, session_id: str) -> dict | None:
with get_db() as conn:
cursor = conn.execute('''
SELECT id, mode, label, started_at, stopped_at, file_path, event_count, size_bytes, metadata
FROM recording_sessions
WHERE id = ?
''', (session_id,))
row = cursor.fetchone()
if not row:
return None
return {
'id': row['id'],
'mode': row['mode'],
'label': row['label'],
'started_at': row['started_at'],
'stopped_at': row['stopped_at'],
'file_path': row['file_path'],
'event_count': row['event_count'],
'size_bytes': row['size_bytes'],
'metadata': json.loads(row['metadata']) if row['metadata'] else {},
}
def get_active(self) -> list[dict]:
with self._lock:
sessions = []
for session in self._active_by_mode.values():
sessions.append({
'id': session.id,
'mode': session.mode,
'label': session.label,
'started_at': session.started_at.isoformat(),
'event_count': session.event_count,
'size_bytes': session.size_bytes,
})
return sessions
_recording_manager: RecordingManager | None = None
_recording_lock = threading.Lock()
def get_recording_manager() -> RecordingManager:
global _recording_manager
with _recording_lock:
if _recording_manager is None:
_recording_manager = RecordingManager()
return _recording_manager
utils/sstv.py
-769
View File
@@ -1,769 +0,0 @@
"""SSTV (Slow-Scan Television) decoder for ISS transmissions.
This module provides SSTV decoding capabilities for receiving images
from the International Space Station during special events.
ISS SSTV typically transmits on 145.800 MHz FM.
Includes real-time Doppler shift compensation for improved reception.
"""
from __future__ import annotations
import os
import queue
import subprocess
import threading
import time
from dataclasses import dataclass, field
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Callable
from utils.logging import get_logger
logger = get_logger('intercept.sstv')
# ISS SSTV frequency
ISS_SSTV_FREQ = 145.800 # MHz
# Speed of light in m/s
SPEED_OF_LIGHT = 299_792_458
# Common SSTV modes used by ISS
SSTV_MODES = ['PD120', 'PD180', 'Martin1', 'Martin2', 'Scottie1', 'Scottie2', 'Robot36']
@dataclass
class DopplerInfo:
"""Doppler shift information."""
frequency_hz: float # Doppler-corrected frequency in Hz
shift_hz: float # Doppler shift in Hz (positive = approaching)
range_rate_km_s: float # Range rate in km/s (negative = approaching)
elevation: float # Current elevation in degrees
azimuth: float # Current azimuth in degrees
timestamp: datetime
def to_dict(self) -> dict:
return {
'frequency_hz': self.frequency_hz,
'shift_hz': round(self.shift_hz, 1),
'range_rate_km_s': round(self.range_rate_km_s, 3),
'elevation': round(self.elevation, 1),
'azimuth': round(self.azimuth, 1),
'timestamp': self.timestamp.isoformat(),
}
class DopplerTracker:
"""
Real-time Doppler shift calculator for satellite tracking.
Uses skyfield to calculate the range rate between observer and satellite,
then computes the Doppler-shifted receive frequency.
"""
def __init__(self, satellite_name: str = 'ISS'):
self._satellite_name = satellite_name
self._observer_lat: float | None = None
self._observer_lon: float | None = None
self._satellite = None
self._observer = None
self._ts = None
self._enabled = False
def configure(self, latitude: float, longitude: float) -> bool:
"""
Configure the Doppler tracker with observer location.
Args:
latitude: Observer latitude in degrees
longitude: Observer longitude in degrees
Returns:
True if configured successfully
"""
try:
from skyfield.api import load, wgs84, EarthSatellite
from data.satellites import TLE_SATELLITES
# Get satellite TLE
tle_data = TLE_SATELLITES.get(self._satellite_name)
if not tle_data:
logger.error(f"No TLE data for satellite: {self._satellite_name}")
return False
self._ts = load.timescale()
self._satellite = EarthSatellite(tle_data[1], tle_data[2], tle_data[0], self._ts)
self._observer = wgs84.latlon(latitude, longitude)
self._observer_lat = latitude
self._observer_lon = longitude
self._enabled = True
logger.info(f"Doppler tracker configured for {self._satellite_name} at ({latitude}, {longitude})")
return True
except ImportError:
logger.warning("skyfield not available - Doppler tracking disabled")
return False
except Exception as e:
logger.error(f"Failed to configure Doppler tracker: {e}")
return False
@property
def is_enabled(self) -> bool:
return self._enabled
def calculate(self, nominal_freq_mhz: float) -> DopplerInfo | None:
"""
Calculate current Doppler-shifted frequency.
Args:
nominal_freq_mhz: Nominal transmit frequency in MHz
Returns:
DopplerInfo with corrected frequency, or None if unavailable
"""
if not self._enabled or not self._satellite or not self._observer:
return None
try:
# Get current time
t = self._ts.now()
# Calculate satellite position relative to observer
difference = self._satellite - self._observer
topocentric = difference.at(t)
# Get altitude/azimuth
alt, az, distance = topocentric.altaz()
# Get velocity (range rate) - negative means approaching
# We need the rate of change of distance
# Calculate positions slightly apart to get velocity
dt_seconds = 1.0
t_future = self._ts.utc(t.utc_datetime() + timedelta(seconds=dt_seconds))
topocentric_future = difference.at(t_future)
_, _, distance_future = topocentric_future.altaz()
# Range rate in km/s (negative = approaching = positive Doppler)
range_rate_km_s = (distance_future.km - distance.km) / dt_seconds
# Calculate Doppler shift
# f_received = f_transmitted * (1 - v_radial / c)
# When approaching (negative range_rate), frequency is higher
nominal_freq_hz = nominal_freq_mhz * 1_000_000
doppler_factor = 1 - (range_rate_km_s * 1000 / SPEED_OF_LIGHT)
corrected_freq_hz = nominal_freq_hz * doppler_factor
shift_hz = corrected_freq_hz - nominal_freq_hz
return DopplerInfo(
frequency_hz=corrected_freq_hz,
shift_hz=shift_hz,
range_rate_km_s=range_rate_km_s,
elevation=alt.degrees,
azimuth=az.degrees,
timestamp=datetime.now(timezone.utc)
)
except Exception as e:
logger.error(f"Doppler calculation failed: {e}")
return None
@dataclass
class SSTVImage:
"""Decoded SSTV image."""
filename: str
path: Path
mode: str
timestamp: datetime
frequency: float
size_bytes: int = 0
def to_dict(self) -> dict:
return {
'filename': self.filename,
'path': str(self.path),
'mode': self.mode,
'timestamp': self.timestamp.isoformat(),
'frequency': self.frequency,
'size_bytes': self.size_bytes,
'url': f'/sstv/images/{self.filename}'
}
@dataclass
class DecodeProgress:
"""SSTV decode progress update."""
status: str # 'detecting', 'decoding', 'complete', 'error'
mode: str | None = None
progress_percent: int = 0
message: str | None = None
image: SSTVImage | None = None
def to_dict(self) -> dict:
result = {
'type': 'sstv_progress',
'status': self.status,
'progress': self.progress_percent,
}
if self.mode:
result['mode'] = self.mode
if self.message:
result['message'] = self.message
if self.image:
result['image'] = self.image.to_dict()
return result
class SSTVDecoder:
"""SSTV decoder using external tools (slowrx) with Doppler compensation."""
# Minimum frequency change (Hz) before retuning rtl_fm
RETUNE_THRESHOLD_HZ = 500
# How often to check/update Doppler (seconds)
DOPPLER_UPDATE_INTERVAL = 5
def __init__(self, output_dir: str | Path | None = None):
self._process = None
self._rtl_process = None
self._running = False
self._lock = threading.Lock()
self._callback: Callable[[DecodeProgress], None] | None = None
self._output_dir = Path(output_dir) if output_dir else Path('instance/sstv_images')
self._images: list[SSTVImage] = []
self._reader_thread = None
self._watcher_thread = None
self._doppler_thread = None
self._frequency = ISS_SSTV_FREQ
self._current_tuned_freq_hz: int = 0
self._device_index = 0
# Doppler tracking
self._doppler_tracker = DopplerTracker('ISS')
self._doppler_enabled = False
self._last_doppler_info: DopplerInfo | None = None
self._file_decoder: str | None = None
# Ensure output directory exists
self._output_dir.mkdir(parents=True, exist_ok=True)
# Detect available decoder
self._decoder = self._detect_decoder()
@property
def is_running(self) -> bool:
return self._running
@property
def decoder_available(self) -> str | None:
"""Return name of available decoder or None."""
return self._decoder
def _detect_decoder(self) -> str | None:
"""Detect which SSTV decoder is available."""
# Check for slowrx (command-line SSTV decoder)
try:
result = subprocess.run(['which', 'slowrx'], capture_output=True, timeout=5)
if result.returncode == 0:
self._file_decoder = 'slowrx'
return 'slowrx'
except Exception:
pass
# Note: qsstv is GUI-only and not suitable for headless/server operation
# Check for Python sstv package
try:
import sstv
self._file_decoder = 'python-sstv'
return None
except ImportError:
pass
logger.warning("No SSTV decoder found. Install slowrx (apt install slowrx) or python sstv package. Note: qsstv is GUI-only and not supported for headless operation.")
return None
def set_callback(self, callback: Callable[[DecodeProgress], None]) -> None:
"""Set callback for decode progress updates."""
self._callback = callback
def start(
self,
frequency: float = ISS_SSTV_FREQ,
device_index: int = 0,
latitude: float | None = None,
longitude: float | None = None,
) -> bool:
"""
Start SSTV decoder listening on specified frequency.
Args:
frequency: Frequency in MHz (default: 145.800 for ISS)
device_index: RTL-SDR device index
latitude: Observer latitude for Doppler correction (optional)
longitude: Observer longitude for Doppler correction (optional)
Returns:
True if started successfully
"""
with self._lock:
if self._running:
return True
if not self._decoder:
logger.error("No SSTV decoder available")
self._emit_progress(DecodeProgress(
status='error',
message='No SSTV decoder installed. Install slowrx: apt install slowrx'
))
return False
self._frequency = frequency
self._device_index = device_index
# Configure Doppler tracking if location provided
self._doppler_enabled = False
if latitude is not None and longitude is not None:
if self._doppler_tracker.configure(latitude, longitude):
self._doppler_enabled = True
logger.info(f"Doppler tracking enabled for location ({latitude}, {longitude})")
else:
logger.warning("Doppler tracking unavailable - using fixed frequency")
try:
if self._decoder == 'slowrx':
self._start_slowrx()
elif self._decoder == 'python-sstv':
self._start_python_sstv()
else:
logger.error(f"Unsupported decoder: {self._decoder}")
return False
self._running = True
# Start Doppler tracking thread if enabled
if self._doppler_enabled:
self._doppler_thread = threading.Thread(target=self._doppler_tracking_loop, daemon=True)
self._doppler_thread.start()
logger.info(f"SSTV decoder started on {frequency} MHz with Doppler tracking")
self._emit_progress(DecodeProgress(
status='detecting',
message=f'Listening on {frequency} MHz with Doppler tracking...'
))
else:
logger.info(f"SSTV decoder started on {frequency} MHz (no Doppler tracking)")
self._emit_progress(DecodeProgress(
status='detecting',
message=f'Listening on {frequency} MHz...'
))
return True
except Exception as e:
logger.error(f"Failed to start SSTV decoder: {e}")
self._emit_progress(DecodeProgress(
status='error',
message=str(e)
))
return False
def _start_slowrx(self) -> None:
"""Start slowrx decoder with rtl_fm piped input."""
# Calculate initial frequency (with Doppler correction if enabled)
freq_hz = self._get_doppler_corrected_freq_hz()
self._current_tuned_freq_hz = freq_hz
self._start_rtl_fm_pipeline(freq_hz)
def _get_doppler_corrected_freq_hz(self) -> int:
"""Get the Doppler-corrected frequency in Hz."""
nominal_freq_hz = int(self._frequency * 1_000_000)
if self._doppler_enabled:
doppler_info = self._doppler_tracker.calculate(self._frequency)
if doppler_info:
self._last_doppler_info = doppler_info
corrected_hz = int(doppler_info.frequency_hz)
logger.info(
f"Doppler correction: {doppler_info.shift_hz:+.1f} Hz "
f"(range rate: {doppler_info.range_rate_km_s:+.3f} km/s, "
f"el: {doppler_info.elevation:.1f}°)"
)
return corrected_hz
return nominal_freq_hz
def _start_rtl_fm_pipeline(self, freq_hz: int) -> None:
"""Start the rtl_fm -> slowrx pipeline at the specified frequency."""
# Build rtl_fm command for FM demodulation
rtl_cmd = [
'rtl_fm',
'-d', str(self._device_index),
'-f', str(freq_hz),
'-M', 'fm',
'-s', '48000',
'-r', '48000',
'-l', '0', # No squelch
'-'
]
# slowrx reads from stdin and outputs images to directory
slowrx_cmd = [
'slowrx',
'-o', str(self._output_dir),
'-'
]
logger.info(f"Starting rtl_fm: {' '.join(rtl_cmd)}")
logger.info(f"Piping to slowrx: {' '.join(slowrx_cmd)}")
# Start rtl_fm
self._rtl_process = subprocess.Popen(
rtl_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
# Start slowrx reading from rtl_fm
self._process = subprocess.Popen(
slowrx_cmd,
stdin=self._rtl_process.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
# Start reader thread to monitor output
self._reader_thread = threading.Thread(target=self._read_slowrx_output, daemon=True)
self._reader_thread.start()
# Start image watcher thread
self._watcher_thread = threading.Thread(target=self._watch_images, daemon=True)
self._watcher_thread.start()
def _doppler_tracking_loop(self) -> None:
"""Background thread that monitors Doppler shift and retunes when needed."""
logger.info("Doppler tracking thread started")
while self._running and self._doppler_enabled:
time.sleep(self.DOPPLER_UPDATE_INTERVAL)
if not self._running:
break
try:
doppler_info = self._doppler_tracker.calculate(self._frequency)
if not doppler_info:
continue
self._last_doppler_info = doppler_info
new_freq_hz = int(doppler_info.frequency_hz)
freq_diff = abs(new_freq_hz - self._current_tuned_freq_hz)
# Log current Doppler status
logger.debug(
f"Doppler: {doppler_info.shift_hz:+.1f} Hz, "
f"el: {doppler_info.elevation:.1f}°, "
f"diff from tuned: {freq_diff} Hz"
)
# Emit Doppler update to callback
self._emit_progress(DecodeProgress(
status='detecting',
message=f'Doppler: {doppler_info.shift_hz:+.0f} Hz, elevation: {doppler_info.elevation:.1f}°'
))
# Retune if frequency has drifted enough
if freq_diff >= self.RETUNE_THRESHOLD_HZ:
logger.info(
f"Retuning: {self._current_tuned_freq_hz} -> {new_freq_hz} Hz "
f"(Doppler shift: {doppler_info.shift_hz:+.1f} Hz)"
)
self._retune_rtl_fm(new_freq_hz)
except Exception as e:
logger.error(f"Doppler tracking error: {e}")
logger.info("Doppler tracking thread stopped")
def _retune_rtl_fm(self, new_freq_hz: int) -> None:
"""
Retune rtl_fm to a new frequency.
Since rtl_fm doesn't support dynamic frequency changes, we need to
restart the rtl_fm process. The slowrx process continues running
and will resume decoding when audio resumes.
"""
with self._lock:
if not self._running:
return
# Terminate old rtl_fm process
if self._rtl_process:
try:
self._rtl_process.terminate()
self._rtl_process.wait(timeout=2)
except Exception:
try:
self._rtl_process.kill()
except Exception:
pass
# Start new rtl_fm at new frequency
rtl_cmd = [
'rtl_fm',
'-d', str(self._device_index),
'-f', str(new_freq_hz),
'-M', 'fm',
'-s', '48000',
'-r', '48000',
'-l', '0',
'-'
]
logger.debug(f"Restarting rtl_fm: {' '.join(rtl_cmd)}")
self._rtl_process = subprocess.Popen(
rtl_cmd,
stdout=self._process.stdin if self._process else subprocess.PIPE,
stderr=subprocess.PIPE
)
self._current_tuned_freq_hz = new_freq_hz
@property
def last_doppler_info(self) -> DopplerInfo | None:
"""Get the most recent Doppler calculation."""
return self._last_doppler_info
@property
def doppler_enabled(self) -> bool:
"""Check if Doppler tracking is enabled."""
return self._doppler_enabled
def _start_python_sstv(self) -> None:
"""Start Python SSTV decoder (requires audio file input)."""
# Python sstv package typically works with audio files
# For real-time decoding, we'd need to record audio first
# This is a simplified implementation
logger.warning("Python SSTV package requires audio file input")
self._emit_progress(DecodeProgress(
status='error',
message='Python SSTV decoder requires audio files. Use slowrx for real-time decoding.'
))
raise NotImplementedError("Real-time Python SSTV not implemented")
def _read_slowrx_output(self) -> None:
"""Read slowrx stderr for progress updates."""
if not self._process:
return
try:
for line in iter(self._process.stderr.readline, b''):
if not self._running:
break
line_str = line.decode('utf-8', errors='ignore').strip()
if not line_str:
continue
logger.debug(f"slowrx: {line_str}")
# Parse slowrx output for mode detection and progress
if 'Detected' in line_str or 'mode' in line_str.lower():
for mode in SSTV_MODES:
if mode.lower() in line_str.lower():
self._emit_progress(DecodeProgress(
status='decoding',
mode=mode,
message=f'Decoding {mode} image...'
))
break
except Exception as e:
logger.error(f"Error reading slowrx output: {e}")
def _watch_images(self) -> None:
"""Watch output directory for new images."""
known_files = set(f.name for f in self._output_dir.glob('*.png'))
while self._running:
time.sleep(1)
try:
current_files = set(f.name for f in self._output_dir.glob('*.png'))
new_files = current_files - known_files
for filename in new_files:
filepath = self._output_dir / filename
if filepath.exists():
# New image detected
image = SSTVImage(
filename=filename,
path=filepath,
mode='Unknown', # Would need to parse from slowrx output
timestamp=datetime.now(timezone.utc),
frequency=self._frequency,
size_bytes=filepath.stat().st_size
)
self._images.append(image)
logger.info(f"New SSTV image: {filename}")
self._emit_progress(DecodeProgress(
status='complete',
message='Image decoded',
image=image
))
known_files = current_files
except Exception as e:
logger.error(f"Error watching images: {e}")
def stop(self) -> None:
"""Stop SSTV decoder."""
with self._lock:
self._running = False
if hasattr(self, '_rtl_process') and self._rtl_process:
try:
self._rtl_process.terminate()
self._rtl_process.wait(timeout=5)
except Exception:
self._rtl_process.kill()
self._rtl_process = None
if self._process:
try:
self._process.terminate()
self._process.wait(timeout=5)
except Exception:
self._process.kill()
self._process = None
logger.info("SSTV decoder stopped")
def get_images(self) -> list[SSTVImage]:
"""Get list of decoded images."""
# Also scan directory for any images we might have missed
self._scan_images()
return list(self._images)
def _scan_images(self) -> None:
"""Scan output directory for images."""
known_filenames = {img.filename for img in self._images}
for filepath in self._output_dir.glob('*.png'):
if filepath.name not in known_filenames:
try:
stat = filepath.stat()
image = SSTVImage(
filename=filepath.name,
path=filepath,
mode='Unknown',
timestamp=datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc),
frequency=ISS_SSTV_FREQ,
size_bytes=stat.st_size
)
self._images.append(image)
except Exception as e:
logger.warning(f"Error scanning image {filepath}: {e}")
def _emit_progress(self, progress: DecodeProgress) -> None:
"""Emit progress update to callback."""
if self._callback:
try:
self._callback(progress)
except Exception as e:
logger.error(f"Error in progress callback: {e}")
def decode_file(self, audio_path: str | Path) -> list[SSTVImage]:
"""
Decode SSTV image from audio file.
Args:
audio_path: Path to WAV audio file
Returns:
List of decoded images
"""
audio_path = Path(audio_path)
if not audio_path.exists():
raise FileNotFoundError(f"Audio file not found: {audio_path}")
images = []
decoder = self._decoder or self._file_decoder
if decoder == 'slowrx':
# Use slowrx with file input
output_file = self._output_dir / f"sstv_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
cmd = ['slowrx', '-o', str(self._output_dir), str(audio_path)]
result = subprocess.run(cmd, capture_output=True, timeout=300)
if result.returncode == 0:
# Check for new images
for filepath in self._output_dir.glob('*.png'):
stat = filepath.stat()
if stat.st_mtime > time.time() - 60: # Created in last minute
image = SSTVImage(
filename=filepath.name,
path=filepath,
mode='Unknown',
timestamp=datetime.now(timezone.utc),
frequency=0,
size_bytes=stat.st_size
)
images.append(image)
elif decoder == 'python-sstv':
# Use Python sstv library
try:
from sstv.decode import SSTVDecoder as PythonSSTVDecoder
from PIL import Image
decoder = PythonSSTVDecoder(str(audio_path))
img = decoder.decode()
if img:
output_file = self._output_dir / f"sstv_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
img.save(output_file)
image = SSTVImage(
filename=output_file.name,
path=output_file,
mode=decoder.mode or 'Unknown',
timestamp=datetime.now(timezone.utc),
frequency=0,
size_bytes=output_file.stat().st_size
)
images.append(image)
except ImportError:
logger.error("Python sstv package not properly installed")
except Exception as e:
logger.error(f"Error decoding with Python sstv: {e}")
return images
# Global decoder instance
_decoder: SSTVDecoder | None = None
def get_sstv_decoder() -> SSTVDecoder:
"""Get or create the global SSTV decoder instance."""
global _decoder
if _decoder is None:
_decoder = SSTVDecoder()
return _decoder
def is_sstv_available() -> bool:
"""Check if SSTV decoding is available."""
decoder = get_sstv_decoder()
return decoder.decoder_available is not None
utils/sstv/__init__.py
+33
View File
@@ -0,0 +1,33 @@
"""SSTV (Slow-Scan Television) decoder package.
Pure Python SSTV decoder using Goertzel-based DSP for VIS header detection
and scanline-by-scanline image decoding. Supports Robot36/72, Martin1/2,
Scottie1/2, and PD120/180 modes.
Replaces the external slowrx dependency with numpy/scipy + Pillow.
"""
from .constants import ISS_SSTV_FREQ, SSTV_MODES
from .sstv_decoder import (
DecodeProgress,
DopplerInfo,
DopplerTracker,
SSTVDecoder,
SSTVImage,
get_general_sstv_decoder,
get_sstv_decoder,
is_sstv_available,
)
__all__ = [
'DecodeProgress',
'DopplerInfo',
'DopplerTracker',
'ISS_SSTV_FREQ',
'SSTV_MODES',
'SSTVDecoder',
'SSTVImage',
'get_general_sstv_decoder',
'get_sstv_decoder',
'is_sstv_available',
]
utils/sstv/constants.py
+92
View File
@@ -0,0 +1,92 @@
"""SSTV protocol constants.
VIS (Vertical Interval Signaling) codes, frequency assignments, and timing
constants for all supported SSTV modes per the SSTV protocol specification.
"""
from __future__ import annotations
# ---------------------------------------------------------------------------
# Audio / DSP
# ---------------------------------------------------------------------------
SAMPLE_RATE = 48000 # Hz - standard audio sample rate used by rtl_fm
# Window size for Goertzel tone detection (5 ms at 48 kHz = 240 samples)
GOERTZEL_WINDOW = 240
# Chunk size for reading from rtl_fm (100 ms = 4800 samples)
STREAM_CHUNK_SAMPLES = 4800
# ---------------------------------------------------------------------------
# SSTV tone frequencies (Hz)
# ---------------------------------------------------------------------------
FREQ_VIS_BIT_1 = 1100 # VIS logic 1
FREQ_SYNC = 1200 # Horizontal sync pulse
FREQ_VIS_BIT_0 = 1300 # VIS logic 0
FREQ_BREAK = 1200 # Break tone in VIS header (same as sync)
FREQ_LEADER = 1900 # Leader / calibration tone
FREQ_BLACK = 1500 # Black level
FREQ_WHITE = 2300 # White level
# Pixel luminance mapping range
FREQ_PIXEL_LOW = 1500 # 0 luminance
FREQ_PIXEL_HIGH = 2300 # 255 luminance
# Frequency tolerance for tone detection (Hz)
FREQ_TOLERANCE = 50
# ---------------------------------------------------------------------------
# VIS header timing (seconds)
# ---------------------------------------------------------------------------
VIS_LEADER_MIN = 0.200 # Minimum leader tone duration
VIS_LEADER_MAX = 0.500 # Maximum leader tone duration
VIS_LEADER_NOMINAL = 0.300 # Nominal leader tone duration
VIS_BREAK_DURATION = 0.010 # Break pulse duration (10 ms)
VIS_BIT_DURATION = 0.030 # Each VIS data bit (30 ms)
VIS_START_BIT_DURATION = 0.030 # Start bit (30 ms)
VIS_STOP_BIT_DURATION = 0.030 # Stop bit (30 ms)
# Timing tolerance for VIS detection
VIS_TIMING_TOLERANCE = 0.5 # 50% tolerance on durations
# ---------------------------------------------------------------------------
# VIS code → mode name mapping
# ---------------------------------------------------------------------------
VIS_CODES: dict[int, str] = {
8: 'Robot36',
12: 'Robot72',
44: 'Martin1',
40: 'Martin2',
60: 'Scottie1',
56: 'Scottie2',
93: 'PD120',
95: 'PD180',
# Less common but recognized
4: 'Robot24',
36: 'Martin3',
52: 'Scottie3',
55: 'ScottieDX',
113: 'PD240',
96: 'PD90',
98: 'PD160',
}
# Reverse mapping: mode name → VIS code
MODE_TO_VIS: dict[str, int] = {v: k for k, v in VIS_CODES.items()}
# ---------------------------------------------------------------------------
# Common SSTV modes list (for UI / status)
# ---------------------------------------------------------------------------
SSTV_MODES = [
'PD120', 'PD180', 'Martin1', 'Martin2',
'Scottie1', 'Scottie2', 'Robot36', 'Robot72',
]
# ISS SSTV frequency
ISS_SSTV_FREQ = 145.800 # MHz
# Speed of light in m/s
SPEED_OF_LIGHT = 299_792_458
# Minimum energy ratio for valid tone detection (vs noise floor)
MIN_ENERGY_RATIO = 5.0
utils/sstv/dsp.py
+232
View File
@@ -0,0 +1,232 @@
"""DSP utilities for SSTV decoding.
Goertzel algorithm for efficient single-frequency energy detection,
frequency estimation, and frequency-to-pixel luminance mapping.
"""
from __future__ import annotations
import math
import numpy as np
from .constants import (
FREQ_PIXEL_HIGH,
FREQ_PIXEL_LOW,
MIN_ENERGY_RATIO,
SAMPLE_RATE,
)
def goertzel(samples: np.ndarray, target_freq: float,
sample_rate: int = SAMPLE_RATE) -> float:
"""Compute Goertzel energy at a single target frequency.
O(N) per frequency - more efficient than FFT when only a few
frequencies are needed.
Args:
samples: Audio samples (float64, -1.0 to 1.0).
target_freq: Frequency to detect (Hz).
sample_rate: Sample rate (Hz).
Returns:
Magnitude squared (energy) at the target frequency.
"""
n = len(samples)
if n == 0:
return 0.0
# Generalized Goertzel (DTFT): use exact target frequency rather than
# rounding to the nearest DFT bin. This is critical for short windows
# (e.g. 13 samples/pixel) where integer-k Goertzel quantizes all SSTV
# pixel frequencies into 1-2 bins, making estimation impossible.
w = 2.0 * math.pi * target_freq / sample_rate
coeff = 2.0 * math.cos(w)
s0 = 0.0
s1 = 0.0
s2 = 0.0
for sample in samples:
s0 = sample + coeff * s1 - s2
s2 = s1
s1 = s0
return s1 * s1 + s2 * s2 - coeff * s1 * s2
def goertzel_mag(samples: np.ndarray, target_freq: float,
sample_rate: int = SAMPLE_RATE) -> float:
"""Compute Goertzel magnitude (square root of energy).
Args:
samples: Audio samples.
target_freq: Frequency to detect (Hz).
sample_rate: Sample rate (Hz).
Returns:
Magnitude at the target frequency.
"""
return math.sqrt(max(0.0, goertzel(samples, target_freq, sample_rate)))
def detect_tone(samples: np.ndarray, candidates: list[float],
sample_rate: int = SAMPLE_RATE) -> tuple[float | None, float]:
"""Detect which candidate frequency has the strongest energy.
Args:
samples: Audio samples.
candidates: List of candidate frequencies (Hz).
sample_rate: Sample rate (Hz).
Returns:
Tuple of (detected_frequency or None, energy_ratio).
Returns None if no tone significantly dominates.
"""
if len(samples) == 0 or not candidates:
return None, 0.0
energies = {f: goertzel(samples, f, sample_rate) for f in candidates}
max_freq = max(energies, key=energies.get) # type: ignore[arg-type]
max_energy = energies[max_freq]
if max_energy <= 0:
return None, 0.0
# Calculate ratio of strongest to average of others
others = [e for f, e in energies.items() if f != max_freq]
avg_others = sum(others) / len(others) if others else 0.0
ratio = max_energy / avg_others if avg_others > 0 else float('inf')
if ratio >= MIN_ENERGY_RATIO:
return max_freq, ratio
return None, ratio
def estimate_frequency(samples: np.ndarray, freq_low: float = 1000.0,
freq_high: float = 2500.0, step: float = 25.0,
sample_rate: int = SAMPLE_RATE) -> float:
"""Estimate the dominant frequency in a range using Goertzel sweep.
Sweeps through frequencies in the given range and returns the one
with maximum energy. Uses a coarse sweep followed by a fine sweep
for accuracy.
Args:
samples: Audio samples.
freq_low: Lower bound of frequency range (Hz).
freq_high: Upper bound of frequency range (Hz).
step: Coarse step size (Hz).
sample_rate: Sample rate (Hz).
Returns:
Estimated dominant frequency (Hz).
"""
if len(samples) == 0:
return 0.0
# Coarse sweep
best_freq = freq_low
best_energy = 0.0
freq = freq_low
while freq <= freq_high:
energy = goertzel(samples, freq, sample_rate)
if energy > best_energy:
best_energy = energy
best_freq = freq
freq += step
# Fine sweep around the coarse peak (+/- one step, 5 Hz resolution)
fine_low = max(freq_low, best_freq - step)
fine_high = min(freq_high, best_freq + step)
freq = fine_low
while freq <= fine_high:
energy = goertzel(samples, freq, sample_rate)
if energy > best_energy:
best_energy = energy
best_freq = freq
freq += 5.0
return best_freq
def freq_to_pixel(frequency: float) -> int:
"""Convert SSTV audio frequency to pixel luminance value (0-255).
Linear mapping: 1500 Hz = 0 (black), 2300 Hz = 255 (white).
Args:
frequency: Detected frequency (Hz).
Returns:
Pixel value clamped to 0-255.
"""
normalized = (frequency - FREQ_PIXEL_LOW) / (FREQ_PIXEL_HIGH - FREQ_PIXEL_LOW)
return max(0, min(255, int(normalized * 255 + 0.5)))
def samples_for_duration(duration_s: float,
sample_rate: int = SAMPLE_RATE) -> int:
"""Calculate number of samples for a given duration.
Args:
duration_s: Duration in seconds.
sample_rate: Sample rate (Hz).
Returns:
Number of samples.
"""
return int(duration_s * sample_rate + 0.5)
def goertzel_batch(audio_matrix: np.ndarray, frequencies: np.ndarray,
sample_rate: int = SAMPLE_RATE) -> np.ndarray:
"""Compute Goertzel energy for multiple audio segments at multiple frequencies.
Vectorized implementation using numpy broadcasting. Processes all
pixel windows and all candidate frequencies simultaneously, giving
roughly 50-100x speed-up over the scalar ``goertzel`` called in a
Python loop.
Args:
audio_matrix: Shape (M, N) – M audio segments of N samples each.
frequencies: 1-D array of F target frequencies in Hz.
sample_rate: Sample rate in Hz.
Returns:
Shape (M, F) array of energy values.
"""
if audio_matrix.size == 0 or len(frequencies) == 0:
return np.zeros((audio_matrix.shape[0], len(frequencies)))
_M, N = audio_matrix.shape
# Generalized Goertzel (DTFT): exact target frequencies, no bin rounding
w = 2.0 * np.pi * frequencies / sample_rate
coeff = 2.0 * np.cos(w) # (F,)
s1 = np.zeros((audio_matrix.shape[0], len(frequencies)))
s2 = np.zeros_like(s1)
for n in range(N):
samples_n = audio_matrix[:, n:n + 1] # (M, 1) — broadcasts with (M, F)
s0 = samples_n + coeff * s1 - s2
s2 = s1
s1 = s0
return s1 * s1 + s2 * s2 - coeff * s1 * s2
def normalize_audio(raw: np.ndarray) -> np.ndarray:
"""Normalize int16 PCM audio to float64 in range [-1.0, 1.0].
Args:
raw: Raw int16 samples from rtl_fm.
Returns:
Float64 normalized samples.
"""
return raw.astype(np.float64) / 32768.0
utils/sstv/image_decoder.py
+460
View File
@@ -0,0 +1,460 @@
"""SSTV scanline-by-scanline image decoder.
Decodes raw audio samples into a PIL Image for all supported SSTV modes.
Handles sync pulse re-synchronization on each line for robust decoding
under weak-signal or drifting conditions.
"""
from __future__ import annotations
from typing import Callable
import numpy as np
from .constants import (
FREQ_BLACK,
FREQ_PIXEL_HIGH,
FREQ_PIXEL_LOW,
FREQ_SYNC,
SAMPLE_RATE,
)
from .dsp import (
goertzel,
goertzel_batch,
samples_for_duration,
)
from .modes import (
ColorModel,
SSTVMode,
SyncPosition,
)
# Pillow is imported lazily to keep the module importable when Pillow
# is not installed (is_sstv_available() just returns True, but actual
# decoding would fail gracefully).
try:
from PIL import Image
except ImportError:
Image = None # type: ignore[assignment,misc]
# Type alias for progress callback: (current_line, total_lines)
ProgressCallback = Callable[[int, int], None]
class SSTVImageDecoder:
"""Decode an SSTV image from a stream of audio samples.
Usage::
decoder = SSTVImageDecoder(mode)
decoder.feed(samples)
...
if decoder.is_complete:
image = decoder.get_image()
"""
def __init__(self, mode: SSTVMode, sample_rate: int = SAMPLE_RATE,
progress_cb: ProgressCallback | None = None):
self._mode = mode
self._sample_rate = sample_rate
self._progress_cb = progress_cb
self._buffer = np.array([], dtype=np.float64)
self._current_line = 0
self._complete = False
# Pre-calculate sample counts
self._sync_samples = samples_for_duration(
mode.sync_duration_ms / 1000.0, sample_rate)
self._porch_samples = samples_for_duration(
mode.sync_porch_ms / 1000.0, sample_rate)
self._line_samples = samples_for_duration(
mode.line_duration_ms / 1000.0, sample_rate)
self._separator_samples = (
samples_for_duration(mode.channel_separator_ms / 1000.0, sample_rate)
if mode.channel_separator_ms > 0 else 0
)
self._channel_samples = [
samples_for_duration(ch.duration_ms / 1000.0, sample_rate)
for ch in mode.channels
]
# For PD modes, each "line" of audio produces 2 image lines
if mode.color_model == ColorModel.YCRCB_DUAL:
self._total_audio_lines = mode.height // 2
else:
self._total_audio_lines = mode.height
# Initialize pixel data arrays per channel
self._channel_data: list[np.ndarray] = []
for _i, _ch_spec in enumerate(mode.channels):
if mode.color_model == ColorModel.YCRCB_DUAL:
# Y1, Cr, Cb, Y2 - all are width-wide
self._channel_data.append(
np.zeros((self._total_audio_lines, mode.width), dtype=np.uint8))
else:
self._channel_data.append(
np.zeros((mode.height, mode.width), dtype=np.uint8))
# Pre-compute candidate frequencies for batch pixel decoding (5 Hz step)
self._freq_candidates = np.arange(
FREQ_PIXEL_LOW - 100, FREQ_PIXEL_HIGH + 105, 5.0)
# Track sync position for re-synchronization
self._expected_line_start = 0 # Sample offset within buffer
self._synced = False
@property
def is_complete(self) -> bool:
return self._complete
@property
def current_line(self) -> int:
return self._current_line
@property
def total_lines(self) -> int:
return self._total_audio_lines
@property
def progress_percent(self) -> int:
if self._total_audio_lines == 0:
return 0
return min(100, int(100 * self._current_line / self._total_audio_lines))
def feed(self, samples: np.ndarray) -> bool:
"""Feed audio samples into the decoder.
Args:
samples: Float64 audio samples.
Returns:
True when image is complete.
"""
if self._complete:
return True
self._buffer = np.concatenate([self._buffer, samples])
# Process complete lines.
# Guard against stalls: if _decode_line() cannot consume data
# (e.g. sub-component samples exceed line_samples due to rounding),
# break out and wait for more audio.
while not self._complete and len(self._buffer) >= self._line_samples:
prev_line = self._current_line
prev_len = len(self._buffer)
self._decode_line()
if self._current_line == prev_line and len(self._buffer) == prev_len:
break # No progress — need more data
# Prevent unbounded buffer growth - keep at most 2 lines worth
max_buffer = self._line_samples * 2
if len(self._buffer) > max_buffer and not self._complete:
self._buffer = self._buffer[-max_buffer:]
return self._complete
def _find_sync(self, search_region: np.ndarray) -> int | None:
"""Find the 1200 Hz sync pulse within a search region.
Scans through the region looking for a stretch of 1200 Hz
tone of approximately the right duration.
Args:
search_region: Audio samples to search within.
Returns:
Sample offset of the sync pulse start, or None if not found.
"""
window_size = min(self._sync_samples, 200)
if len(search_region) < window_size:
return None
best_pos = None
best_energy = 0.0
step = window_size // 2
for pos in range(0, len(search_region) - window_size, step):
chunk = search_region[pos:pos + window_size]
sync_energy = goertzel(chunk, FREQ_SYNC, self._sample_rate)
# Check it's actually sync, not data at 1200 Hz area
black_energy = goertzel(chunk, FREQ_BLACK, self._sample_rate)
if sync_energy > best_energy and sync_energy > black_energy * 2:
best_energy = sync_energy
best_pos = pos
return best_pos
def _decode_line(self) -> None:
"""Decode one scanline from the buffer."""
if self._current_line >= self._total_audio_lines:
self._complete = True
return
# Try to find sync pulse for re-synchronization
# Search within +/-10% of expected line start
search_margin = max(100, self._line_samples // 10)
line_start = 0
if self._mode.sync_position in (SyncPosition.FRONT, SyncPosition.FRONT_PD):
# Sync is at the beginning of each line
search_start = 0
search_end = min(len(self._buffer), self._sync_samples + search_margin)
search_region = self._buffer[search_start:search_end]
sync_pos = self._find_sync(search_region)
if sync_pos is not None:
line_start = sync_pos
# Skip sync + porch to get to pixel data
pixel_start = line_start + self._sync_samples + self._porch_samples
elif self._mode.sync_position == SyncPosition.MIDDLE:
# Scottie: sep(1.5ms) -> G -> sep(1.5ms) -> B -> sync(9ms) -> porch(1.5ms) -> R
# Skip initial separator (same duration as porch)
pixel_start = self._porch_samples
line_start = 0
else:
pixel_start = self._sync_samples + self._porch_samples
# Decode each channel
pos = pixel_start
for ch_idx, ch_samples in enumerate(self._channel_samples):
if pos + ch_samples > len(self._buffer):
# Not enough data yet - put the data back and wait
return
channel_audio = self._buffer[pos:pos + ch_samples]
pixels = self._decode_channel_pixels(channel_audio)
self._channel_data[ch_idx][self._current_line, :] = pixels
pos += ch_samples
# Add inter-channel gaps based on mode family
if ch_idx < len(self._channel_samples) - 1:
if self._mode.sync_position == SyncPosition.MIDDLE:
if ch_idx == 0:
# Scottie: separator between G and B
pos += self._porch_samples
else:
# Scottie: sync + porch between B and R
pos += self._sync_samples + self._porch_samples
elif self._separator_samples > 0:
# Robot: separator + porch between channels
pos += self._separator_samples
elif (self._mode.sync_position == SyncPosition.FRONT
and self._mode.color_model == ColorModel.RGB):
# Martin: porch between channels
pos += self._porch_samples
# Advance buffer past this line
consumed = max(pos, self._line_samples)
self._buffer = self._buffer[consumed:]
self._current_line += 1
if self._progress_cb:
self._progress_cb(self._current_line, self._total_audio_lines)
if self._current_line >= self._total_audio_lines:
self._complete = True
# Minimum analysis window for meaningful Goertzel frequency estimation.
# With 96 samples (2ms at 48kHz), frequency accuracy is within ~25 Hz,
# giving pixel-level accuracy of ~8/255 levels.
_MIN_ANALYSIS_WINDOW = 96
def _decode_channel_pixels(self, audio: np.ndarray) -> np.ndarray:
"""Decode pixel values from a channel's audio data.
Uses batch Goertzel to estimate frequencies for all pixels
simultaneously, then maps to luminance values. When pixels have
fewer samples than ``_MIN_ANALYSIS_WINDOW``, overlapping analysis
windows are used to maintain frequency estimation accuracy.
Args:
audio: Audio samples for one channel of one scanline.
Returns:
Array of pixel values (0-255), shape (width,).
"""
width = self._mode.width
samples_per_pixel = max(1, len(audio) // width)
if len(audio) < width or samples_per_pixel < 2:
return np.zeros(width, dtype=np.uint8)
window_size = max(samples_per_pixel, self._MIN_ANALYSIS_WINDOW)
if window_size > samples_per_pixel and len(audio) >= window_size:
# Use overlapping windows centered on each pixel position
windows = np.lib.stride_tricks.sliding_window_view(
audio, window_size)
# Pixel centers, clamped to valid window indices
centers = np.arange(width) * samples_per_pixel
indices = np.minimum(centers, len(windows) - 1)
audio_matrix = np.ascontiguousarray(windows[indices])
else:
# Non-overlapping: each pixel has enough samples
usable = width * samples_per_pixel
audio_matrix = audio[:usable].reshape(width, samples_per_pixel)
# Batch Goertzel at all candidate frequencies
energies = goertzel_batch(
audio_matrix, self._freq_candidates, self._sample_rate)
# Find peak frequency per pixel
best_idx = np.argmax(energies, axis=1)
best_freqs = self._freq_candidates[best_idx]
# Map frequencies to pixel values (1500 Hz = 0, 2300 Hz = 255)
normalized = (best_freqs - FREQ_PIXEL_LOW) / (FREQ_PIXEL_HIGH - FREQ_PIXEL_LOW)
return np.clip(normalized * 255 + 0.5, 0, 255).astype(np.uint8)
def get_image(self) -> Image.Image | None:
"""Convert decoded channel data to a PIL Image.
Returns:
PIL Image in RGB mode, or None if Pillow is not available
or decoding is incomplete.
"""
if Image is None:
return None
mode = self._mode
if mode.color_model == ColorModel.RGB:
return self._assemble_rgb()
elif mode.color_model == ColorModel.YCRCB:
return self._assemble_ycrcb()
elif mode.color_model == ColorModel.YCRCB_DUAL:
return self._assemble_ycrcb_dual()
return None
def _assemble_rgb(self) -> Image.Image:
"""Assemble RGB image from sequential R, G, B channel data.
Martin/Scottie channel order: G, B, R.
"""
height = self._mode.height
# Channel order for Martin/Scottie: [0]=G, [1]=B, [2]=R
g_data = self._channel_data[0][:height]
b_data = self._channel_data[1][:height]
r_data = self._channel_data[2][:height]
rgb = np.stack([r_data, g_data, b_data], axis=-1)
return Image.fromarray(rgb, 'RGB')
def _assemble_ycrcb(self) -> Image.Image:
"""Assemble image from YCrCb data (Robot modes).
Robot36: Y every line, Cr/Cb alternating (half-rate chroma).
Robot72: Y, Cr, Cb every line (full-rate chroma).
"""
height = self._mode.height
width = self._mode.width
if not self._mode.has_half_rate_chroma:
# Full-rate chroma (Robot72): Y, Cr, Cb as separate channels
y_data = self._channel_data[0][:height].astype(np.float64)
cr = self._channel_data[1][:height].astype(np.float64)
cb = self._channel_data[2][:height].astype(np.float64)
return self._ycrcb_to_rgb(y_data, cr, cb, height, width)
# Half-rate chroma (Robot36): Y + alternating Cr/Cb
y_data = self._channel_data[0][:height].astype(np.float64)
chroma_data = self._channel_data[1][:height].astype(np.float64)
# Separate Cr (even lines) and Cb (odd lines), then interpolate
cr = np.zeros((height, width), dtype=np.float64)
cb = np.zeros((height, width), dtype=np.float64)
for line in range(height):
if line % 2 == 0:
cr[line] = chroma_data[line]
else:
cb[line] = chroma_data[line]
# Interpolate missing chroma lines
for line in range(height):
if line % 2 == 1:
# Missing Cr - interpolate from neighbors
prev_cr = line - 1 if line > 0 else line + 1
next_cr = line + 1 if line + 1 < height else line - 1
cr[line] = (cr[prev_cr] + cr[next_cr]) / 2
else:
# Missing Cb - interpolate from neighbors
prev_cb = line - 1 if line > 0 else line + 1
next_cb = line + 1 if line + 1 < height else line - 1
if prev_cb >= 0 and next_cb < height:
cb[line] = (cb[prev_cb] + cb[next_cb]) / 2
elif prev_cb >= 0:
cb[line] = cb[prev_cb]
else:
cb[line] = cb[next_cb]
return self._ycrcb_to_rgb(y_data, cr, cb, height, width)
def _assemble_ycrcb_dual(self) -> Image.Image:
"""Assemble image from dual-luminance YCrCb data (PD modes).
PD modes send Y1, Cr, Cb, Y2 per audio line, producing 2 image lines.
"""
audio_lines = self._total_audio_lines
width = self._mode.width
height = self._mode.height
y1_data = self._channel_data[0][:audio_lines].astype(np.float64)
cr_data = self._channel_data[1][:audio_lines].astype(np.float64)
cb_data = self._channel_data[2][:audio_lines].astype(np.float64)
y2_data = self._channel_data[3][:audio_lines].astype(np.float64)
# Interleave Y1 and Y2 to produce full-height luminance
y_full = np.zeros((height, width), dtype=np.float64)
cr_full = np.zeros((height, width), dtype=np.float64)
cb_full = np.zeros((height, width), dtype=np.float64)
for i in range(audio_lines):
even_line = i * 2
odd_line = i * 2 + 1
if even_line < height:
y_full[even_line] = y1_data[i]
cr_full[even_line] = cr_data[i]
cb_full[even_line] = cb_data[i]
if odd_line < height:
y_full[odd_line] = y2_data[i]
cr_full[odd_line] = cr_data[i]
cb_full[odd_line] = cb_data[i]
return self._ycrcb_to_rgb(y_full, cr_full, cb_full, height, width)
@staticmethod
def _ycrcb_to_rgb(y: np.ndarray, cr: np.ndarray, cb: np.ndarray,
height: int, width: int) -> Image.Image:
"""Convert YCrCb pixel data to an RGB PIL Image.
Uses the SSTV convention where pixel values 0-255 map to the
standard Y'CbCr color space used by JPEG/SSTV.
"""
# Normalize from 0-255 pixel range to standard ranges
# Y: 0-255, Cr/Cb: 0-255 centered at 128
y_norm = y
cr_norm = cr - 128.0
cb_norm = cb - 128.0
# ITU-R BT.601 conversion
r = y_norm + 1.402 * cr_norm
g = y_norm - 0.344136 * cb_norm - 0.714136 * cr_norm
b = y_norm + 1.772 * cb_norm
# Clip and convert
r = np.clip(r, 0, 255).astype(np.uint8)
g = np.clip(g, 0, 255).astype(np.uint8)
b = np.clip(b, 0, 255).astype(np.uint8)
rgb = np.stack([r, g, b], axis=-1)
return Image.fromarray(rgb, 'RGB')
utils/sstv/modes.py
+250
View File
@@ -0,0 +1,250 @@
"""SSTV mode specifications.
Dataclass definitions for each supported SSTV mode, encoding resolution,
color model, line timing, and sync characteristics.
"""
from __future__ import annotations
import enum
from dataclasses import dataclass, field
class ColorModel(enum.Enum):
"""Color encoding models used by SSTV modes."""
RGB = 'rgb' # Sequential R, G, B channels per line
YCRCB = 'ycrcb' # Luminance + chrominance (Robot modes)
YCRCB_DUAL = 'ycrcb_dual' # Dual-luminance YCrCb (PD modes)
class SyncPosition(enum.Enum):
"""Where the horizontal sync pulse appears in each line."""
FRONT = 'front' # Sync at start of line (Robot, Martin)
MIDDLE = 'middle' # Sync between G and B channels (Scottie)
FRONT_PD = 'front_pd' # PD-style sync at start
@dataclass(frozen=True)
class ChannelTiming:
"""Timing for a single color channel within a scanline.
Attributes:
duration_ms: Duration of this channel's pixel data in milliseconds.
"""
duration_ms: float
@dataclass(frozen=True)
class SSTVMode:
"""Complete specification of an SSTV mode.
Attributes:
name: Human-readable mode name (e.g. 'Robot36').
vis_code: VIS code that identifies this mode.
width: Image width in pixels.
height: Image height in lines.
color_model: Color encoding model.
sync_position: Where the sync pulse falls in each line.
sync_duration_ms: Horizontal sync pulse duration (ms).
sync_porch_ms: Porch (gap) after sync pulse (ms).
channels: Timing for each color channel per line.
line_duration_ms: Total duration of one complete scanline (ms).
has_half_rate_chroma: Whether chroma is sent at half vertical rate
(Robot modes: Cr and Cb alternate every other line).
"""
name: str
vis_code: int
width: int
height: int
color_model: ColorModel
sync_position: SyncPosition
sync_duration_ms: float
sync_porch_ms: float
channels: list[ChannelTiming] = field(default_factory=list)
line_duration_ms: float = 0.0
has_half_rate_chroma: bool = False
channel_separator_ms: float = 0.0 # Time gap between color channels (ms)
# ---------------------------------------------------------------------------
# Robot family
# ---------------------------------------------------------------------------
ROBOT_36 = SSTVMode(
name='Robot36',
vis_code=8,
width=320,
height=240,
color_model=ColorModel.YCRCB,
sync_position=SyncPosition.FRONT,
sync_duration_ms=9.0,
sync_porch_ms=3.0,
channels=[
ChannelTiming(duration_ms=88.0), # Y (luminance)
ChannelTiming(duration_ms=44.0), # Cr or Cb (alternating)
],
line_duration_ms=150.0,
has_half_rate_chroma=True,
channel_separator_ms=6.0,
)
ROBOT_72 = SSTVMode(
name='Robot72',
vis_code=12,
width=320,
height=240,
color_model=ColorModel.YCRCB,
sync_position=SyncPosition.FRONT,
sync_duration_ms=9.0,
sync_porch_ms=3.0,
channels=[
ChannelTiming(duration_ms=138.0), # Y (luminance)
ChannelTiming(duration_ms=69.0), # Cr
ChannelTiming(duration_ms=69.0), # Cb
],
line_duration_ms=300.0,
has_half_rate_chroma=False,
channel_separator_ms=6.0,
)
# ---------------------------------------------------------------------------
# Martin family
# ---------------------------------------------------------------------------
MARTIN_1 = SSTVMode(
name='Martin1',
vis_code=44,
width=320,
height=256,
color_model=ColorModel.RGB,
sync_position=SyncPosition.FRONT,
sync_duration_ms=4.862,
sync_porch_ms=0.572,
channels=[
ChannelTiming(duration_ms=146.432), # Green
ChannelTiming(duration_ms=146.432), # Blue
ChannelTiming(duration_ms=146.432), # Red
],
line_duration_ms=446.446,
)
MARTIN_2 = SSTVMode(
name='Martin2',
vis_code=40,
width=320,
height=256,
color_model=ColorModel.RGB,
sync_position=SyncPosition.FRONT,
sync_duration_ms=4.862,
sync_porch_ms=0.572,
channels=[
ChannelTiming(duration_ms=73.216), # Green
ChannelTiming(duration_ms=73.216), # Blue
ChannelTiming(duration_ms=73.216), # Red
],
line_duration_ms=226.798,
)
# ---------------------------------------------------------------------------
# Scottie family
# ---------------------------------------------------------------------------
SCOTTIE_1 = SSTVMode(
name='Scottie1',
vis_code=60,
width=320,
height=256,
color_model=ColorModel.RGB,
sync_position=SyncPosition.MIDDLE,
sync_duration_ms=9.0,
sync_porch_ms=1.5,
channels=[
ChannelTiming(duration_ms=138.240), # Green
ChannelTiming(duration_ms=138.240), # Blue
ChannelTiming(duration_ms=138.240), # Red
],
line_duration_ms=428.220,
)
SCOTTIE_2 = SSTVMode(
name='Scottie2',
vis_code=56,
width=320,
height=256,
color_model=ColorModel.RGB,
sync_position=SyncPosition.MIDDLE,
sync_duration_ms=9.0,
sync_porch_ms=1.5,
channels=[
ChannelTiming(duration_ms=88.064), # Green
ChannelTiming(duration_ms=88.064), # Blue
ChannelTiming(duration_ms=88.064), # Red
],
line_duration_ms=277.692,
)
# ---------------------------------------------------------------------------
# PD (Pasokon) family
# ---------------------------------------------------------------------------
PD_120 = SSTVMode(
name='PD120',
vis_code=93,
width=640,
height=496,
color_model=ColorModel.YCRCB_DUAL,
sync_position=SyncPosition.FRONT_PD,
sync_duration_ms=20.0,
sync_porch_ms=2.080,
channels=[
ChannelTiming(duration_ms=121.600), # Y1 (even line luminance)
ChannelTiming(duration_ms=121.600), # Cr
ChannelTiming(duration_ms=121.600), # Cb
ChannelTiming(duration_ms=121.600), # Y2 (odd line luminance)
],
line_duration_ms=508.480,
)
PD_180 = SSTVMode(
name='PD180',
vis_code=95,
width=640,
height=496,
color_model=ColorModel.YCRCB_DUAL,
sync_position=SyncPosition.FRONT_PD,
sync_duration_ms=20.0,
sync_porch_ms=2.080,
channels=[
ChannelTiming(duration_ms=183.040), # Y1
ChannelTiming(duration_ms=183.040), # Cr
ChannelTiming(duration_ms=183.040), # Cb
ChannelTiming(duration_ms=183.040), # Y2
],
line_duration_ms=754.240,
)
# ---------------------------------------------------------------------------
# Mode registry
# ---------------------------------------------------------------------------
ALL_MODES: dict[int, SSTVMode] = {
m.vis_code: m for m in [
ROBOT_36, ROBOT_72,
MARTIN_1, MARTIN_2,
SCOTTIE_1, SCOTTIE_2,
PD_120, PD_180,
]
}
MODE_BY_NAME: dict[str, SSTVMode] = {m.name: m for m in ALL_MODES.values()}
def get_mode(vis_code: int) -> SSTVMode | None:
"""Look up an SSTV mode by its VIS code."""
return ALL_MODES.get(vis_code)
def get_mode_by_name(name: str) -> SSTVMode | None:
"""Look up an SSTV mode by name."""
return MODE_BY_NAME.get(name)
utils/sstv/sstv_decoder.py
+905
View File
@@ -0,0 +1,905 @@
"""SSTV decoder orchestrator.
Provides the SSTVDecoder class that manages the full pipeline:
rtl_fm subprocess -> audio stream -> VIS detection -> image decoding -> PNG output.
Also contains DopplerTracker and supporting dataclasses migrated from the
original monolithic utils/sstv.py.
"""
from __future__ import annotations
import base64
import contextlib
import io
import subprocess
import threading
import time
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Callable
import numpy as np
from utils.logging import get_logger
from .constants import ISS_SSTV_FREQ, SAMPLE_RATE, SPEED_OF_LIGHT
from .dsp import goertzel_mag, normalize_audio
from .image_decoder import SSTVImageDecoder
from .modes import get_mode
from .vis import VISDetector
logger = get_logger('intercept.sstv')
try:
from PIL import Image as PILImage
except ImportError:
PILImage = None # type: ignore[assignment,misc]
# ---------------------------------------------------------------------------
# Dataclasses
# ---------------------------------------------------------------------------
@dataclass
class DopplerInfo:
"""Doppler shift information."""
frequency_hz: float
shift_hz: float
range_rate_km_s: float
elevation: float
azimuth: float
timestamp: datetime
def to_dict(self) -> dict:
return {
'frequency_hz': self.frequency_hz,
'shift_hz': round(self.shift_hz, 1),
'range_rate_km_s': round(self.range_rate_km_s, 3),
'elevation': round(self.elevation, 1),
'azimuth': round(self.azimuth, 1),
'timestamp': self.timestamp.isoformat(),
}
@dataclass
class SSTVImage:
"""Decoded SSTV image."""
filename: str
path: Path
mode: str
timestamp: datetime
frequency: float
size_bytes: int = 0
url_prefix: str = '/sstv'
def to_dict(self) -> dict:
return {
'filename': self.filename,
'path': str(self.path),
'mode': self.mode,
'timestamp': self.timestamp.isoformat(),
'frequency': self.frequency,
'size_bytes': self.size_bytes,
'url': f'{self.url_prefix}/images/{self.filename}'
}
@dataclass
class DecodeProgress:
"""SSTV decode progress update."""
status: str # 'detecting', 'decoding', 'complete', 'error'
mode: str | None = None
progress_percent: int = 0
message: str | None = None
image: SSTVImage | None = None
signal_level: int | None = None # 0-100 RMS audio level, None = not measured
sstv_tone: str | None = None # 'leader', 'sync', 'noise', None
vis_state: str | None = None # VIS detector state name
partial_image: str | None = None # base64 data URL of partial decode
def to_dict(self) -> dict:
result: dict = {
'type': 'sstv_progress',
'status': self.status,
'progress': self.progress_percent,
}
if self.mode:
result['mode'] = self.mode
if self.message:
result['message'] = self.message
if self.image:
result['image'] = self.image.to_dict()
if self.signal_level is not None:
result['signal_level'] = self.signal_level
if self.sstv_tone:
result['sstv_tone'] = self.sstv_tone
if self.vis_state:
result['vis_state'] = self.vis_state
if self.partial_image:
result['partial_image'] = self.partial_image
return result
# ---------------------------------------------------------------------------
# DopplerTracker
# ---------------------------------------------------------------------------
class DopplerTracker:
"""Real-time Doppler shift calculator for satellite tracking.
Uses skyfield to calculate the range rate between observer and satellite,
then computes the Doppler-shifted receive frequency.
"""
def __init__(self, satellite_name: str = 'ISS'):
self._satellite_name = satellite_name
self._observer_lat: float | None = None
self._observer_lon: float | None = None
self._satellite = None
self._observer = None
self._ts = None
self._enabled = False
def configure(self, latitude: float, longitude: float) -> bool:
"""Configure the Doppler tracker with observer location."""
try:
from skyfield.api import EarthSatellite, load, wgs84
from data.satellites import TLE_SATELLITES
tle_data = TLE_SATELLITES.get(self._satellite_name)
if not tle_data:
logger.error(f"No TLE data for satellite: {self._satellite_name}")
return False
self._ts = load.timescale()
self._satellite = EarthSatellite(tle_data[1], tle_data[2], tle_data[0], self._ts)
self._observer = wgs84.latlon(latitude, longitude)
self._observer_lat = latitude
self._observer_lon = longitude
self._enabled = True
logger.info(f"Doppler tracker configured for {self._satellite_name} at ({latitude}, {longitude})")
return True
except ImportError:
logger.warning("skyfield not available - Doppler tracking disabled")
return False
except Exception as e:
logger.error(f"Failed to configure Doppler tracker: {e}")
return False
@property
def is_enabled(self) -> bool:
return self._enabled
def calculate(self, nominal_freq_mhz: float) -> DopplerInfo | None:
"""Calculate current Doppler-shifted frequency."""
if not self._enabled or not self._satellite or not self._observer:
return None
try:
t = self._ts.now()
difference = self._satellite - self._observer
topocentric = difference.at(t)
alt, az, distance = topocentric.altaz()
dt_seconds = 1.0
t_future = self._ts.utc(t.utc_datetime() + timedelta(seconds=dt_seconds))
topocentric_future = difference.at(t_future)
_, _, distance_future = topocentric_future.altaz()
range_rate_km_s = (distance_future.km - distance.km) / dt_seconds
nominal_freq_hz = nominal_freq_mhz * 1_000_000
doppler_factor = 1 - (range_rate_km_s * 1000 / SPEED_OF_LIGHT)
corrected_freq_hz = nominal_freq_hz * doppler_factor
shift_hz = corrected_freq_hz - nominal_freq_hz
return DopplerInfo(
frequency_hz=corrected_freq_hz,
shift_hz=shift_hz,
range_rate_km_s=range_rate_km_s,
elevation=alt.degrees,
azimuth=az.degrees,
timestamp=datetime.now(timezone.utc)
)
except Exception as e:
logger.error(f"Doppler calculation failed: {e}")
return None
# ---------------------------------------------------------------------------
# SSTVDecoder
# ---------------------------------------------------------------------------
class SSTVDecoder:
"""SSTV decoder using pure-Python DSP with Doppler compensation."""
RETUNE_THRESHOLD_HZ = 500
DOPPLER_UPDATE_INTERVAL = 5
def __init__(self, output_dir: str | Path | None = None, url_prefix: str = '/sstv'):
self._rtl_process = None
self._running = False
self._lock = threading.Lock()
self._callback: Callable[[DecodeProgress], None] | None = None
self._output_dir = Path(output_dir) if output_dir else Path('instance/sstv_images')
self._url_prefix = url_prefix
self._images: list[SSTVImage] = []
self._decode_thread = None
self._doppler_thread = None
self._frequency = ISS_SSTV_FREQ
self._modulation = 'fm'
self._current_tuned_freq_hz: int = 0
self._device_index = 0
# Doppler tracking
self._doppler_tracker = DopplerTracker('ISS')
self._doppler_enabled = False
self._last_doppler_info: DopplerInfo | None = None
# Ensure output directory exists
self._output_dir.mkdir(parents=True, exist_ok=True)
@property
def is_running(self) -> bool:
return self._running
@property
def decoder_available(self) -> str:
"""Return name of available decoder. Always available with pure Python."""
return 'python-sstv'
def set_callback(self, callback: Callable[[DecodeProgress], None]) -> None:
"""Set callback for decode progress updates."""
self._callback = callback
def start(
self,
frequency: float = ISS_SSTV_FREQ,
device_index: int = 0,
latitude: float | None = None,
longitude: float | None = None,
modulation: str = 'fm',
) -> bool:
"""Start SSTV decoder listening on specified frequency.
Args:
frequency: Frequency in MHz (default: 145.800 for ISS).
device_index: RTL-SDR device index.
latitude: Observer latitude for Doppler correction.
longitude: Observer longitude for Doppler correction.
modulation: Demodulation mode for rtl_fm (fm, usb, lsb).
Returns:
True if started successfully.
"""
with self._lock:
if self._running:
return True
self._frequency = frequency
self._device_index = device_index
self._modulation = modulation
# Configure Doppler tracking if location provided
self._doppler_enabled = False
if latitude is not None and longitude is not None:
if self._doppler_tracker.configure(latitude, longitude):
self._doppler_enabled = True
logger.info(f"Doppler tracking enabled for location ({latitude}, {longitude})")
else:
logger.warning("Doppler tracking unavailable - using fixed frequency")
try:
freq_hz = self._get_doppler_corrected_freq_hz()
self._current_tuned_freq_hz = freq_hz
# Set _running BEFORE starting the pipeline so the decode
# thread sees it as True on its first loop iteration.
self._running = True
self._start_pipeline(freq_hz)
# Start Doppler tracking thread if enabled
if self._doppler_enabled:
self._doppler_thread = threading.Thread(
target=self._doppler_tracking_loop, daemon=True)
self._doppler_thread.start()
logger.info(f"SSTV decoder started on {frequency} MHz with Doppler tracking")
self._emit_progress(DecodeProgress(
status='detecting',
message=f'Listening on {frequency} MHz with Doppler tracking...'
))
else:
logger.info(f"SSTV decoder started on {frequency} MHz (no Doppler tracking)")
self._emit_progress(DecodeProgress(
status='detecting',
message=f'Listening on {frequency} MHz...'
))
return True
except Exception as e:
self._running = False
logger.error(f"Failed to start SSTV decoder: {e}")
self._emit_progress(DecodeProgress(
status='error',
message=str(e)
))
return False
def _get_doppler_corrected_freq_hz(self) -> int:
"""Get the Doppler-corrected frequency in Hz."""
nominal_freq_hz = int(self._frequency * 1_000_000)
if self._doppler_enabled:
doppler_info = self._doppler_tracker.calculate(self._frequency)
if doppler_info:
self._last_doppler_info = doppler_info
corrected_hz = int(doppler_info.frequency_hz)
logger.info(
f"Doppler correction: {doppler_info.shift_hz:+.1f} Hz "
f"(range rate: {doppler_info.range_rate_km_s:+.3f} km/s, "
f"el: {doppler_info.elevation:.1f}\u00b0)"
)
return corrected_hz
return nominal_freq_hz
def _start_pipeline(self, freq_hz: int) -> None:
"""Start the rtl_fm -> Python decode pipeline."""
rtl_cmd = [
'rtl_fm',
'-d', str(self._device_index),
'-f', str(freq_hz),
'-M', self._modulation,
'-s', str(SAMPLE_RATE),
'-r', str(SAMPLE_RATE),
'-l', '0', # No squelch
'-'
]
logger.info(f"Starting rtl_fm: {' '.join(rtl_cmd)}")
self._rtl_process = subprocess.Popen(
rtl_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
# Start decode thread that reads from rtl_fm stdout
self._decode_thread = threading.Thread(
target=self._decode_audio_stream, daemon=True)
self._decode_thread.start()
def _decode_audio_stream(self) -> None:
"""Read audio from rtl_fm and decode SSTV images.
Runs in a background thread. Reads 100ms chunks of int16 PCM,
feeds through VIS detector, then image decoder.
"""
chunk_bytes = SAMPLE_RATE // 10 * 2 # 100ms of int16 = 9600 bytes
vis_detector = VISDetector(sample_rate=SAMPLE_RATE)
image_decoder: SSTVImageDecoder | None = None
current_mode_name: str | None = None
chunk_counter = 0
last_partial_pct = -1
logger.info("Audio decode thread started")
rtl_fm_error: str = ''
while self._running and self._rtl_process:
try:
raw_data = self._rtl_process.stdout.read(chunk_bytes)
if not raw_data:
if self._running:
# Read stderr to diagnose why rtl_fm exited
stderr_msg = ''
if self._rtl_process and self._rtl_process.stderr:
with contextlib.suppress(Exception):
stderr_msg = self._rtl_process.stderr.read().decode(
errors='replace').strip()
rc = self._rtl_process.poll() if self._rtl_process else None
logger.warning(
f"rtl_fm stream ended unexpectedly "
f"(exit code: {rc})"
)
if stderr_msg:
logger.warning(f"rtl_fm stderr: {stderr_msg}")
rtl_fm_error = stderr_msg
break
# Convert int16 PCM to float64
n_samples = len(raw_data) // 2
if n_samples == 0:
continue
raw_samples = np.frombuffer(raw_data[:n_samples * 2], dtype=np.int16)
samples = normalize_audio(raw_samples)
chunk_counter += 1
if image_decoder is not None:
# Currently decoding an image
complete = image_decoder.feed(samples)
# Encode partial image every 5% progress
pct = image_decoder.progress_percent
partial_url = None
if pct >= last_partial_pct + 5 or complete:
last_partial_pct = pct
try:
img = image_decoder.get_image()
if img is not None:
buf = io.BytesIO()
img.save(buf, format='JPEG', quality=40)
b64 = base64.b64encode(buf.getvalue()).decode('ascii')
partial_url = f'data:image/jpeg;base64,{b64}'
except Exception:
pass
# Emit progress
self._emit_progress(DecodeProgress(
status='decoding',
mode=current_mode_name,
progress_percent=pct,
message=f'Decoding {current_mode_name}: {pct}%',
partial_image=partial_url,
))
if complete:
# Save image
self._save_decoded_image(image_decoder, current_mode_name)
image_decoder = None
current_mode_name = None
vis_detector.reset()
else:
# Scanning for VIS header
result = vis_detector.feed(samples)
if result is not None:
vis_code, mode_name = result
logger.info(f"VIS detected: code={vis_code}, mode={mode_name}")
mode_spec = get_mode(vis_code)
if mode_spec:
current_mode_name = mode_name
image_decoder = SSTVImageDecoder(
mode_spec,
sample_rate=SAMPLE_RATE,
)
self._emit_progress(DecodeProgress(
status='decoding',
mode=mode_name,
progress_percent=0,
message=f'Detected {mode_name} - decoding...'
))
else:
logger.warning(f"No mode spec for VIS code {vis_code}")
vis_detector.reset()
# Emit signal level metrics every ~500ms (every 5th 100ms chunk)
if chunk_counter % 5 == 0 and image_decoder is None:
rms = float(np.sqrt(np.mean(samples ** 2)))
signal_level = min(100, int(rms * 500))
leader_energy = goertzel_mag(samples, 1900.0, SAMPLE_RATE)
sync_energy = goertzel_mag(samples, 1200.0, SAMPLE_RATE)
noise_floor = max(rms * 0.5, 0.001)
# Require the tone to both exceed the noise floor AND
# dominate the other tone by 2x to avoid false positives
# from broadband noise.
if (leader_energy > noise_floor * 5
and leader_energy > sync_energy * 2):
sstv_tone = 'leader'
elif (sync_energy > noise_floor * 5
and sync_energy > leader_energy * 2):
sstv_tone = 'sync'
elif signal_level > 10:
sstv_tone = 'noise'
else:
sstv_tone = None
self._emit_progress(DecodeProgress(
status='detecting',
message='Listening...',
signal_level=signal_level,
sstv_tone=sstv_tone,
vis_state=vis_detector.state.value,
))
except Exception as e:
logger.error(f"Error in decode thread: {e}")
if not self._running:
break
time.sleep(0.1)
# Clean up if the thread exits while we thought we were running.
# This prevents a "ghost running" state where is_running is True
# but the thread has already died (e.g. rtl_fm exited).
with self._lock:
was_running = self._running
self._running = False
if was_running and self._rtl_process:
with contextlib.suppress(Exception):
self._rtl_process.terminate()
self._rtl_process.wait(timeout=2)
self._rtl_process = None
if was_running:
logger.warning("Audio decode thread stopped unexpectedly")
err_detail = rtl_fm_error.split('\n')[-1] if rtl_fm_error else ''
msg = f'rtl_fm failed: {err_detail}' if err_detail else 'Decode pipeline stopped unexpectedly'
self._emit_progress(DecodeProgress(
status='error',
message=msg
))
else:
logger.info("Audio decode thread stopped")
def _save_decoded_image(self, decoder: SSTVImageDecoder,
mode_name: str | None) -> None:
"""Save a completed decoded image to disk."""
try:
img = decoder.get_image()
if img is None:
logger.error("Failed to get image from decoder (Pillow not available?)")
self._emit_progress(DecodeProgress(
status='error',
message='Failed to create image - Pillow not installed'
))
return
timestamp = datetime.now(timezone.utc)
filename = f"sstv_{timestamp.strftime('%Y%m%d_%H%M%S')}_{mode_name or 'unknown'}.png"
filepath = self._output_dir / filename
img.save(filepath, 'PNG')
sstv_image = SSTVImage(
filename=filename,
path=filepath,
mode=mode_name or 'Unknown',
timestamp=timestamp,
frequency=self._frequency,
size_bytes=filepath.stat().st_size,
url_prefix=self._url_prefix,
)
self._images.append(sstv_image)
logger.info(f"SSTV image saved: {filename} ({sstv_image.size_bytes} bytes)")
self._emit_progress(DecodeProgress(
status='complete',
mode=mode_name,
progress_percent=100,
message='Image decoded',
image=sstv_image,
))
except Exception as e:
logger.error(f"Error saving decoded image: {e}")
self._emit_progress(DecodeProgress(
status='error',
message=f'Error saving image: {e}'
))
def _doppler_tracking_loop(self) -> None:
"""Background thread that monitors Doppler shift and retunes when needed."""
logger.info("Doppler tracking thread started")
while self._running and self._doppler_enabled:
time.sleep(self.DOPPLER_UPDATE_INTERVAL)
if not self._running:
break
try:
doppler_info = self._doppler_tracker.calculate(self._frequency)
if not doppler_info:
continue
self._last_doppler_info = doppler_info
new_freq_hz = int(doppler_info.frequency_hz)
freq_diff = abs(new_freq_hz - self._current_tuned_freq_hz)
logger.debug(
f"Doppler: {doppler_info.shift_hz:+.1f} Hz, "
f"el: {doppler_info.elevation:.1f}\u00b0, "
f"diff from tuned: {freq_diff} Hz"
)
self._emit_progress(DecodeProgress(
status='detecting',
message=f'Doppler: {doppler_info.shift_hz:+.0f} Hz, elevation: {doppler_info.elevation:.1f}\u00b0'
))
if freq_diff >= self.RETUNE_THRESHOLD_HZ:
logger.info(
f"Retuning: {self._current_tuned_freq_hz} -> {new_freq_hz} Hz "
f"(Doppler shift: {doppler_info.shift_hz:+.1f} Hz)"
)
self._retune_rtl_fm(new_freq_hz)
except Exception as e:
logger.error(f"Doppler tracking error: {e}")
logger.info("Doppler tracking thread stopped")
def _retune_rtl_fm(self, new_freq_hz: int) -> None:
"""Retune rtl_fm to a new frequency by restarting the process."""
with self._lock:
if not self._running:
return
if self._rtl_process:
try:
self._rtl_process.terminate()
self._rtl_process.wait(timeout=2)
except Exception:
with contextlib.suppress(Exception):
self._rtl_process.kill()
rtl_cmd = [
'rtl_fm',
'-d', str(self._device_index),
'-f', str(new_freq_hz),
'-M', self._modulation,
'-s', str(SAMPLE_RATE),
'-r', str(SAMPLE_RATE),
'-l', '0',
'-'
]
logger.debug(f"Restarting rtl_fm: {' '.join(rtl_cmd)}")
self._rtl_process = subprocess.Popen(
rtl_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
self._current_tuned_freq_hz = new_freq_hz
@property
def last_doppler_info(self) -> DopplerInfo | None:
"""Get the most recent Doppler calculation."""
return self._last_doppler_info
@property
def doppler_enabled(self) -> bool:
"""Check if Doppler tracking is enabled."""
return self._doppler_enabled
def stop(self) -> None:
"""Stop SSTV decoder."""
with self._lock:
self._running = False
if self._rtl_process:
try:
self._rtl_process.terminate()
self._rtl_process.wait(timeout=5)
except Exception:
with contextlib.suppress(Exception):
self._rtl_process.kill()
self._rtl_process = None
logger.info("SSTV decoder stopped")
def get_images(self) -> list[SSTVImage]:
"""Get list of decoded images."""
self._scan_images()
return list(self._images)
def delete_image(self, filename: str) -> bool:
"""Delete a single decoded image by filename."""
filepath = self._output_dir / filename
if not filepath.exists():
return False
filepath.unlink()
self._images = [img for img in self._images if img.filename != filename]
logger.info(f"Deleted SSTV image: {filename}")
return True
def delete_all_images(self) -> int:
"""Delete all decoded images. Returns count deleted."""
count = 0
for filepath in self._output_dir.glob('*.png'):
filepath.unlink()
count += 1
self._images.clear()
logger.info(f"Deleted all SSTV images ({count} files)")
return count
def _scan_images(self) -> None:
"""Scan output directory for images."""
known_filenames = {img.filename for img in self._images}
for filepath in self._output_dir.glob('*.png'):
if filepath.name not in known_filenames:
try:
stat = filepath.stat()
image = SSTVImage(
filename=filepath.name,
path=filepath,
mode='Unknown',
timestamp=datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc),
frequency=self._frequency,
size_bytes=stat.st_size,
url_prefix=self._url_prefix,
)
self._images.append(image)
except Exception as e:
logger.warning(f"Error scanning image {filepath}: {e}")
def _emit_progress(self, progress: DecodeProgress) -> None:
"""Emit progress update to callback."""
if self._callback:
try:
self._callback(progress)
except Exception as e:
logger.error(f"Error in progress callback: {e}")
def decode_file(self, audio_path: str | Path) -> list[SSTVImage]:
"""Decode SSTV image(s) from an audio file.
Reads a WAV file and processes it through VIS detection + image
decoding using the pure Python pipeline.
Args:
audio_path: Path to WAV audio file.
Returns:
List of decoded images.
"""
import wave
audio_path = Path(audio_path)
if not audio_path.exists():
raise FileNotFoundError(f"Audio file not found: {audio_path}")
images: list[SSTVImage] = []
try:
with wave.open(str(audio_path), 'rb') as wf:
n_channels = wf.getnchannels()
sample_width = wf.getsampwidth()
file_sample_rate = wf.getframerate()
n_frames = wf.getnframes()
logger.info(
f"Decoding WAV: {n_channels}ch, {sample_width*8}bit, "
f"{file_sample_rate}Hz, {n_frames} frames"
)
# Read all audio data
raw_data = wf.readframes(n_frames)
# Convert to float64 mono
if sample_width == 2:
audio = np.frombuffer(raw_data, dtype=np.int16).astype(np.float64) / 32768.0
elif sample_width == 1:
audio = np.frombuffer(raw_data, dtype=np.uint8).astype(np.float64) / 128.0 - 1.0
elif sample_width == 4:
audio = np.frombuffer(raw_data, dtype=np.int32).astype(np.float64) / 2147483648.0
else:
raise ValueError(f"Unsupported sample width: {sample_width}")
# If stereo, take left channel
if n_channels > 1:
audio = audio[::n_channels]
# Resample if needed
if file_sample_rate != SAMPLE_RATE:
audio = self._resample(audio, file_sample_rate, SAMPLE_RATE)
# Process through VIS detector + image decoder
vis_detector = VISDetector(sample_rate=SAMPLE_RATE)
image_decoder: SSTVImageDecoder | None = None
current_mode_name: str | None = None
chunk_size = SAMPLE_RATE // 10 # 100ms chunks
offset = 0
while offset < len(audio):
chunk = audio[offset:offset + chunk_size]
offset += chunk_size
if image_decoder is not None:
complete = image_decoder.feed(chunk)
if complete:
img = image_decoder.get_image()
if img is not None:
timestamp = datetime.now(timezone.utc)
filename = f"sstv_{timestamp.strftime('%Y%m%d_%H%M%S')}_{current_mode_name or 'unknown'}.png"
filepath = self._output_dir / filename
img.save(filepath, 'PNG')
sstv_image = SSTVImage(
filename=filename,
path=filepath,
mode=current_mode_name or 'Unknown',
timestamp=timestamp,
frequency=0,
size_bytes=filepath.stat().st_size,
url_prefix=self._url_prefix,
)
images.append(sstv_image)
self._images.append(sstv_image)
logger.info(f"Decoded image from file: {filename}")
image_decoder = None
current_mode_name = None
vis_detector.reset()
else:
result = vis_detector.feed(chunk)
if result is not None:
vis_code, mode_name = result
logger.info(f"VIS detected in file: code={vis_code}, mode={mode_name}")
mode_spec = get_mode(vis_code)
if mode_spec:
current_mode_name = mode_name
image_decoder = SSTVImageDecoder(
mode_spec,
sample_rate=SAMPLE_RATE,
)
else:
vis_detector.reset()
except wave.Error as e:
logger.error(f"Error reading WAV file: {e}")
raise
except Exception as e:
logger.error(f"Error decoding audio file: {e}")
raise
return images
@staticmethod
def _resample(audio: np.ndarray, from_rate: int, to_rate: int) -> np.ndarray:
"""Simple resampling using linear interpolation."""
if from_rate == to_rate:
return audio
ratio = to_rate / from_rate
new_length = int(len(audio) * ratio)
indices = np.linspace(0, len(audio) - 1, new_length)
return np.interp(indices, np.arange(len(audio)), audio)
# ---------------------------------------------------------------------------
# Module-level singletons
# ---------------------------------------------------------------------------
_decoder: SSTVDecoder | None = None
def get_sstv_decoder() -> SSTVDecoder:
"""Get or create the global SSTV decoder instance."""
global _decoder
if _decoder is None:
_decoder = SSTVDecoder()
return _decoder
def is_sstv_available() -> bool:
"""Check if SSTV decoding is available.
Always True with the pure-Python decoder (requires only numpy/Pillow).
"""
return True
_general_decoder: SSTVDecoder | None = None
def get_general_sstv_decoder() -> SSTVDecoder:
"""Get or create the global general SSTV decoder instance."""
global _general_decoder
if _general_decoder is None:
_general_decoder = SSTVDecoder(
output_dir='instance/sstv_general_images',
url_prefix='/sstv-general',
)
return _general_decoder
utils/sstv/vis.py
+324
View File
@@ -0,0 +1,324 @@
"""VIS (Vertical Interval Signaling) header detection.
State machine that processes audio samples to detect the VIS header
that precedes every SSTV image transmission. The VIS header identifies
the SSTV mode (Robot36, Martin1, etc.) via an 8-bit code with even parity.
VIS header structure:
Leader tone (1900 Hz, ~300ms)
Break (1200 Hz, ~10ms)
Leader tone (1900 Hz, ~300ms)
Start bit (1200 Hz, 30ms)
8 data bits (1100 Hz = 1, 1300 Hz = 0, 30ms each)
Parity bit (even parity, 30ms)
Stop bit (1200 Hz, 30ms)
"""
from __future__ import annotations
import enum
import numpy as np
from .constants import (
FREQ_LEADER,
FREQ_SYNC,
FREQ_VIS_BIT_0,
FREQ_VIS_BIT_1,
SAMPLE_RATE,
VIS_BIT_DURATION,
VIS_CODES,
VIS_LEADER_MAX,
VIS_LEADER_MIN,
)
from .dsp import goertzel, samples_for_duration
# Use 10ms window (480 samples at 48kHz) for 100Hz frequency resolution.
# This cleanly separates 1100, 1200, 1300, 1500, 1900, 2300 Hz tones.
VIS_WINDOW = 480
class VISState(enum.Enum):
"""States of the VIS detection state machine."""
IDLE = 'idle'
LEADER_1 = 'leader_1'
BREAK = 'break'
LEADER_2 = 'leader_2'
START_BIT = 'start_bit'
DATA_BITS = 'data_bits'
PARITY = 'parity'
STOP_BIT = 'stop_bit'
DETECTED = 'detected'
# The four tone classes we need to distinguish in VIS detection.
_VIS_FREQS = [FREQ_VIS_BIT_1, FREQ_SYNC, FREQ_VIS_BIT_0, FREQ_LEADER]
# 1100, 1200, 1300, 1900 Hz
def _classify_tone(samples: np.ndarray,
sample_rate: int = SAMPLE_RATE) -> float | None:
"""Classify which VIS tone is present in the given samples.
Computes Goertzel energy at each of the four VIS frequencies and returns
the one with the highest energy, provided it dominates sufficiently.
Returns:
The detected frequency (1100, 1200, 1300, or 1900), or None.
"""
if len(samples) < 16:
return None
energies = {f: goertzel(samples, f, sample_rate) for f in _VIS_FREQS}
best_freq = max(energies, key=energies.get) # type: ignore[arg-type]
best_energy = energies[best_freq]
if best_energy <= 0:
return None
# Require the best frequency to be at least 2x stronger than the
# next-strongest tone.
others = sorted(
[e for f, e in energies.items() if f != best_freq], reverse=True)
second_best = others[0] if others else 0.0
if second_best > 0 and best_energy / second_best < 2.0:
return None
return best_freq
class VISDetector:
"""VIS header detection state machine.
Feed audio samples via ``feed()`` and it returns the detected VIS code
(and mode name) when a valid header is found.
The state machine uses a simple approach:
- **Leader detection**: Count consecutive 1900 Hz windows until minimum
leader duration is met.
- **Break/start bit**: Count consecutive 1200 Hz windows. The break is
short; the start bit is one VIS bit duration.
- **Data/parity bits**: Accumulate audio for one bit duration, then
compare 1100 vs 1300 Hz energy to determine bit value.
- **Stop bit**: Count 1200 Hz windows for one bit duration.
Usage::
detector = VISDetector()
for chunk in audio_chunks:
result = detector.feed(chunk)
if result is not None:
vis_code, mode_name = result
"""
def __init__(self, sample_rate: int = SAMPLE_RATE):
self._sample_rate = sample_rate
self._window = VIS_WINDOW
self._bit_samples = samples_for_duration(VIS_BIT_DURATION, sample_rate)
self._leader_min_samples = samples_for_duration(VIS_LEADER_MIN, sample_rate)
self._leader_max_samples = samples_for_duration(VIS_LEADER_MAX, sample_rate)
# Pre-calculate window counts
self._leader_min_windows = max(1, self._leader_min_samples // self._window)
self._leader_max_windows = max(1, self._leader_max_samples // self._window)
self._bit_windows = max(1, self._bit_samples // self._window)
self._state = VISState.IDLE
self._buffer = np.array([], dtype=np.float64)
self._tone_counter = 0
self._data_bits: list[int] = []
self._parity_bit: int = 0
self._bit_accumulator: list[np.ndarray] = []
def reset(self) -> None:
"""Reset the detector to scan for a new VIS header."""
self._state = VISState.IDLE
self._buffer = np.array([], dtype=np.float64)
self._tone_counter = 0
self._data_bits = []
self._parity_bit = 0
self._bit_accumulator = []
@property
def state(self) -> VISState:
return self._state
def feed(self, samples: np.ndarray) -> tuple[int, str] | None:
"""Feed audio samples and attempt VIS detection.
Args:
samples: Float64 audio samples (normalized to -1..1).
Returns:
(vis_code, mode_name) tuple when a valid VIS header is detected,
or None if still scanning.
"""
self._buffer = np.concatenate([self._buffer, samples])
while len(self._buffer) >= self._window:
result = self._process_window(self._buffer[:self._window])
self._buffer = self._buffer[self._window:]
if result is not None:
return result
return None
def _process_window(self, window: np.ndarray) -> tuple[int, str] | None:
"""Process a single analysis window through the state machine.
The key design: when a state transition occurs due to a tone change,
the window that triggers the transition counts as the first window
of the new state (tone_counter = 1).
"""
tone = _classify_tone(window, self._sample_rate)
if self._state == VISState.IDLE:
if tone == FREQ_LEADER:
self._tone_counter += 1
if self._tone_counter >= self._leader_min_windows:
self._state = VISState.LEADER_1
else:
self._tone_counter = 0
elif self._state == VISState.LEADER_1:
if tone == FREQ_LEADER:
self._tone_counter += 1
if self._tone_counter > self._leader_max_windows * 3:
self._tone_counter = 0
self._state = VISState.IDLE
elif tone == FREQ_SYNC:
# Transition to BREAK; this window counts as break window 1
self._tone_counter = 1
self._state = VISState.BREAK
elif tone is None:
pass # Ambiguous window at tone boundary — stay in state
else:
self._tone_counter = 0
self._state = VISState.IDLE
elif self._state == VISState.BREAK:
if tone == FREQ_SYNC:
self._tone_counter += 1
if self._tone_counter > 10:
self._tone_counter = 0
self._state = VISState.IDLE
elif tone == FREQ_LEADER:
# Transition to LEADER_2; this window counts
self._tone_counter = 1
self._state = VISState.LEADER_2
elif tone is None:
pass # Ambiguous window at tone boundary — stay in state
else:
self._tone_counter = 0
self._state = VISState.IDLE
elif self._state == VISState.LEADER_2:
if tone == FREQ_LEADER:
self._tone_counter += 1
if self._tone_counter > self._leader_max_windows * 3:
self._tone_counter = 0
self._state = VISState.IDLE
elif tone == FREQ_SYNC:
# Transition to START_BIT; this window counts
self._tone_counter = 1
self._state = VISState.START_BIT
# Check if start bit is already complete (1-window bit)
if self._tone_counter >= self._bit_windows:
self._tone_counter = 0
self._data_bits = []
self._bit_accumulator = []
self._state = VISState.DATA_BITS
elif tone is None:
pass # Ambiguous window at tone boundary — stay in state
else:
self._tone_counter = 0
self._state = VISState.IDLE
elif self._state == VISState.START_BIT:
if tone == FREQ_SYNC:
self._tone_counter += 1
if self._tone_counter >= self._bit_windows:
self._tone_counter = 0
self._data_bits = []
self._bit_accumulator = []
self._state = VISState.DATA_BITS
else:
# Non-sync during start bit: check if we had enough sync
# windows already (tolerant: accept if within 1 window)
if self._tone_counter >= self._bit_windows - 1:
# Close enough - accept and process this window as data
self._data_bits = []
self._bit_accumulator = [window]
self._tone_counter = 1
self._state = VISState.DATA_BITS
else:
self._tone_counter = 0
self._state = VISState.IDLE
elif self._state == VISState.DATA_BITS:
self._tone_counter += 1
self._bit_accumulator.append(window)
if self._tone_counter >= self._bit_windows:
bit_audio = np.concatenate(self._bit_accumulator)
bit_val = self._decode_bit(bit_audio)
self._data_bits.append(bit_val)
self._tone_counter = 0
self._bit_accumulator = []
if len(self._data_bits) == 8:
self._state = VISState.PARITY
elif self._state == VISState.PARITY:
self._tone_counter += 1
self._bit_accumulator.append(window)
if self._tone_counter >= self._bit_windows:
bit_audio = np.concatenate(self._bit_accumulator)
self._parity_bit = self._decode_bit(bit_audio)
self._tone_counter = 0
self._bit_accumulator = []
self._state = VISState.STOP_BIT
elif self._state == VISState.STOP_BIT:
self._tone_counter += 1
if self._tone_counter >= self._bit_windows:
result = self._validate_and_decode()
self.reset()
return result
return None
def _decode_bit(self, samples: np.ndarray) -> int:
"""Decode a single VIS data bit from its audio samples.
Compares Goertzel energy at 1100 Hz (bit=1) vs 1300 Hz (bit=0).
"""
e1 = goertzel(samples, FREQ_VIS_BIT_1, self._sample_rate)
e0 = goertzel(samples, FREQ_VIS_BIT_0, self._sample_rate)
return 1 if e1 > e0 else 0
def _validate_and_decode(self) -> tuple[int, str] | None:
"""Validate parity and decode the VIS code.
Returns:
(vis_code, mode_name) or None if validation fails.
"""
if len(self._data_bits) != 8:
return None
# Decode VIS code (LSB first)
vis_code = 0
for i, bit in enumerate(self._data_bits):
vis_code |= bit << i
# Look up mode
mode_name = VIS_CODES.get(vis_code)
if mode_name is not None:
return vis_code, mode_name
return None
utils/tscm/advanced.py
+74 -22
View File
@@ -523,20 +523,22 @@ class BaselineDiff:
}
def calculate_baseline_diff(
baseline: dict,
current_wifi: list[dict],
current_bt: list[dict],
current_rf: list[dict],
sweep_id: int
) -> BaselineDiff:
def calculate_baseline_diff(
baseline: dict,
current_wifi: list[dict],
current_wifi_clients: list[dict],
current_bt: list[dict],
current_rf: list[dict],
sweep_id: int
) -> BaselineDiff:
"""
Calculate comprehensive diff between baseline and current scan.
Args:
baseline: Baseline dict from database
current_wifi: Current WiFi devices
current_bt: Current Bluetooth devices
current_wifi_clients: Current WiFi clients
current_bt: Current Bluetooth devices
current_rf: Current RF signals
sweep_id: Current sweep ID
@@ -564,11 +566,16 @@ def calculate_baseline_diff(
diff.is_stale = diff.baseline_age_hours > 72
# Build baseline lookup dicts
baseline_wifi = {
d.get('bssid', d.get('mac', '')).upper(): d
for d in baseline.get('wifi_networks', [])
if d.get('bssid') or d.get('mac')
}
baseline_wifi = {
d.get('bssid', d.get('mac', '')).upper(): d
for d in baseline.get('wifi_networks', [])
if d.get('bssid') or d.get('mac')
}
baseline_wifi_clients = {
d.get('mac', d.get('address', '')).upper(): d
for d in baseline.get('wifi_clients', [])
if d.get('mac') or d.get('address')
}
baseline_bt = {
d.get('mac', d.get('address', '')).upper(): d
for d in baseline.get('bt_devices', [])
@@ -580,8 +587,11 @@ def calculate_baseline_diff(
if d.get('frequency')
}
# Compare WiFi
_compare_wifi(diff, baseline_wifi, current_wifi)
# Compare WiFi
_compare_wifi(diff, baseline_wifi, current_wifi)
# Compare WiFi clients
_compare_wifi_clients(diff, baseline_wifi_clients, current_wifi_clients)
# Compare Bluetooth
_compare_bluetooth(diff, baseline_bt, current_bt)
@@ -607,7 +617,7 @@ def calculate_baseline_diff(
return diff
def _compare_wifi(diff: BaselineDiff, baseline: dict, current: list[dict]) -> None:
def _compare_wifi(diff: BaselineDiff, baseline: dict, current: list[dict]) -> None:
"""Compare WiFi devices between baseline and current."""
current_macs = {
d.get('bssid', d.get('mac', '')).upper(): d
@@ -630,7 +640,48 @@ def _compare_wifi(diff: BaselineDiff, baseline: dict, current: list[dict]) -> No
'channel': device.get('channel'),
'rssi': device.get('power', device.get('signal')),
}
))
))
def _compare_wifi_clients(diff: BaselineDiff, baseline: dict, current: list[dict]) -> None:
"""Compare WiFi clients between baseline and current."""
current_macs = {
d.get('mac', d.get('address', '')).upper(): d
for d in current
if d.get('mac') or d.get('address')
}
# Find new clients
for mac, device in current_macs.items():
if mac not in baseline:
name = device.get('vendor', 'WiFi Client')
diff.new_devices.append(DeviceChange(
identifier=mac,
protocol='wifi_client',
change_type='new',
description=f'New WiFi client: {name}',
expected=False,
details={
'vendor': name,
'rssi': device.get('rssi'),
'associated_bssid': device.get('associated_bssid'),
}
))
# Find missing clients
for mac, device in baseline.items():
if mac not in current_macs:
name = device.get('vendor', 'WiFi Client')
diff.missing_devices.append(DeviceChange(
identifier=mac,
protocol='wifi_client',
change_type='missing',
description=f'Missing WiFi client: {name}',
expected=True,
details={
'vendor': name,
}
))
else:
# Check for changes
baseline_dev = baseline[mac]
@@ -796,11 +847,12 @@ def _calculate_baseline_health(diff: BaselineDiff, baseline: dict) -> None:
reasons.append(f"Baseline is {diff.baseline_age_hours:.0f} hours old")
# Device churn penalty
total_baseline = (
len(baseline.get('wifi_networks', [])) +
len(baseline.get('bt_devices', [])) +
len(baseline.get('rf_frequencies', []))
)
total_baseline = (
len(baseline.get('wifi_networks', [])) +
len(baseline.get('wifi_clients', [])) +
len(baseline.get('bt_devices', [])) +
len(baseline.get('rf_frequencies', []))
)
if total_baseline > 0:
churn_rate = (diff.total_new + diff.total_missing) / total_baseline
utils/tscm/baseline.py
+161 -84
View File
@@ -26,12 +26,13 @@ class BaselineRecorder:
Records and manages TSCM environment baselines.
"""
def __init__(self):
self.recording = False
self.current_baseline_id: int | None = None
self.wifi_networks: dict[str, dict] = {} # BSSID -> network info
self.bt_devices: dict[str, dict] = {} # MAC -> device info
self.rf_frequencies: dict[float, dict] = {} # Frequency -> signal info
def __init__(self):
self.recording = False
self.current_baseline_id: int | None = None
self.wifi_networks: dict[str, dict] = {} # BSSID -> network info
self.wifi_clients: dict[str, dict] = {} # MAC -> client info
self.bt_devices: dict[str, dict] = {} # MAC -> device info
self.rf_frequencies: dict[float, dict] = {} # Frequency -> signal info
def start_recording(
self,
@@ -50,10 +51,11 @@ class BaselineRecorder:
Returns:
Baseline ID
"""
self.recording = True
self.wifi_networks = {}
self.bt_devices = {}
self.rf_frequencies = {}
self.recording = True
self.wifi_networks = {}
self.wifi_clients = {}
self.bt_devices = {}
self.rf_frequencies = {}
# Create baseline in database
self.current_baseline_id = create_tscm_baseline(
@@ -78,24 +80,27 @@ class BaselineRecorder:
self.recording = False
# Convert to lists for storage
wifi_list = list(self.wifi_networks.values())
bt_list = list(self.bt_devices.values())
rf_list = list(self.rf_frequencies.values())
wifi_list = list(self.wifi_networks.values())
wifi_client_list = list(self.wifi_clients.values())
bt_list = list(self.bt_devices.values())
rf_list = list(self.rf_frequencies.values())
# Update database
update_tscm_baseline(
self.current_baseline_id,
wifi_networks=wifi_list,
bt_devices=bt_list,
rf_frequencies=rf_list
)
update_tscm_baseline(
self.current_baseline_id,
wifi_networks=wifi_list,
wifi_clients=wifi_client_list,
bt_devices=bt_list,
rf_frequencies=rf_list
)
summary = {
'baseline_id': self.current_baseline_id,
'wifi_count': len(wifi_list),
'bt_count': len(bt_list),
'rf_count': len(rf_list),
}
summary = {
'baseline_id': self.current_baseline_id,
'wifi_count': len(wifi_list),
'wifi_client_count': len(wifi_client_list),
'bt_count': len(bt_list),
'rf_count': len(rf_list),
}
logger.info(
f"Baseline recording complete: {summary['wifi_count']} WiFi, "
@@ -135,8 +140,8 @@ class BaselineRecorder:
'last_seen': datetime.now().isoformat(),
}
def add_bt_device(self, device: dict) -> None:
"""Add a Bluetooth device to the current baseline."""
def add_bt_device(self, device: dict) -> None:
"""Add a Bluetooth device to the current baseline."""
if not self.recording:
return
@@ -150,7 +155,7 @@ class BaselineRecorder:
'rssi': device.get('rssi', self.bt_devices[mac].get('rssi')),
})
else:
self.bt_devices[mac] = {
self.bt_devices[mac] = {
'mac': mac,
'name': device.get('name', ''),
'rssi': device.get('rssi', device.get('signal')),
@@ -158,10 +163,37 @@ class BaselineRecorder:
'type': device.get('type', ''),
'first_seen': datetime.now().isoformat(),
'last_seen': datetime.now().isoformat(),
}
def add_rf_signal(self, signal: dict) -> None:
"""Add an RF signal to the current baseline."""
}
def add_wifi_client(self, client: dict) -> None:
"""Add a WiFi client to the current baseline."""
if not self.recording:
return
mac = client.get('mac', client.get('address', '')).upper()
if not mac:
return
if mac in self.wifi_clients:
self.wifi_clients[mac].update({
'last_seen': datetime.now().isoformat(),
'rssi': client.get('rssi', self.wifi_clients[mac].get('rssi')),
'associated_bssid': client.get('associated_bssid', self.wifi_clients[mac].get('associated_bssid')),
})
else:
self.wifi_clients[mac] = {
'mac': mac,
'vendor': client.get('vendor', ''),
'rssi': client.get('rssi'),
'associated_bssid': client.get('associated_bssid'),
'probed_ssids': client.get('probed_ssids', []),
'probe_count': client.get('probe_count', len(client.get('probed_ssids', []))),
'first_seen': datetime.now().isoformat(),
'last_seen': datetime.now().isoformat(),
}
def add_rf_signal(self, signal: dict) -> None:
"""Add an RF signal to the current baseline."""
if not self.recording:
return
@@ -191,15 +223,16 @@ class BaselineRecorder:
'hit_count': 1,
}
def get_recording_status(self) -> dict:
"""Get current recording status and counts."""
return {
'recording': self.recording,
'baseline_id': self.current_baseline_id,
'wifi_count': len(self.wifi_networks),
'bt_count': len(self.bt_devices),
'rf_count': len(self.rf_frequencies),
}
def get_recording_status(self) -> dict:
"""Get current recording status and counts."""
return {
'recording': self.recording,
'baseline_id': self.current_baseline_id,
'wifi_count': len(self.wifi_networks),
'wifi_client_count': len(self.wifi_clients),
'bt_count': len(self.bt_devices),
'rf_count': len(self.rf_frequencies),
}
class BaselineComparator:
@@ -220,11 +253,16 @@ class BaselineComparator:
for d in baseline.get('wifi_networks', [])
if d.get('bssid') or d.get('mac')
}
self.baseline_bt = {
d.get('mac', d.get('address', '')).upper(): d
for d in baseline.get('bt_devices', [])
if d.get('mac') or d.get('address')
}
self.baseline_bt = {
d.get('mac', d.get('address', '')).upper(): d
for d in baseline.get('bt_devices', [])
if d.get('mac') or d.get('address')
}
self.baseline_wifi_clients = {
d.get('mac', d.get('address', '')).upper(): d
for d in baseline.get('wifi_clients', [])
if d.get('mac') or d.get('address')
}
self.baseline_rf = {
round(d.get('frequency', 0), 1): d
for d in baseline.get('rf_frequencies', [])
@@ -269,8 +307,8 @@ class BaselineComparator:
'matching_count': len(matching_devices),
}
def compare_bluetooth(self, current_devices: list[dict]) -> dict:
"""Compare current Bluetooth devices against baseline."""
def compare_bluetooth(self, current_devices: list[dict]) -> dict:
"""Compare current Bluetooth devices against baseline."""
current_macs = {
d.get('mac', d.get('address', '')).upper(): d
for d in current_devices
@@ -291,14 +329,45 @@ class BaselineComparator:
if mac not in current_macs:
missing_devices.append(device)
return {
'new': new_devices,
'missing': missing_devices,
'matching': matching_devices,
'new_count': len(new_devices),
'missing_count': len(missing_devices),
'matching_count': len(matching_devices),
}
return {
'new': new_devices,
'missing': missing_devices,
'matching': matching_devices,
'new_count': len(new_devices),
'missing_count': len(missing_devices),
'matching_count': len(matching_devices),
}
def compare_wifi_clients(self, current_devices: list[dict]) -> dict:
"""Compare current WiFi clients against baseline."""
current_macs = {
d.get('mac', d.get('address', '')).upper(): d
for d in current_devices
if d.get('mac') or d.get('address')
}
new_devices = []
missing_devices = []
matching_devices = []
for mac, device in current_macs.items():
if mac not in self.baseline_wifi_clients:
new_devices.append(device)
else:
matching_devices.append(device)
for mac, device in self.baseline_wifi_clients.items():
if mac not in current_macs:
missing_devices.append(device)
return {
'new': new_devices,
'missing': missing_devices,
'matching': matching_devices,
'new_count': len(new_devices),
'missing_count': len(missing_devices),
'matching_count': len(matching_devices),
}
def compare_rf(self, current_signals: list[dict]) -> dict:
"""Compare current RF signals against baseline."""
@@ -331,35 +400,42 @@ class BaselineComparator:
'matching_count': len(matching_signals),
}
def compare_all(
self,
wifi_devices: list[dict] | None = None,
bt_devices: list[dict] | None = None,
rf_signals: list[dict] | None = None
) -> dict:
def compare_all(
self,
wifi_devices: list[dict] | None = None,
wifi_clients: list[dict] | None = None,
bt_devices: list[dict] | None = None,
rf_signals: list[dict] | None = None
) -> dict:
"""
Compare all current data against baseline.
Returns:
Dict with comparison results for each category
"""
results = {
'wifi': None,
'bluetooth': None,
'rf': None,
'total_new': 0,
'total_missing': 0,
}
results = {
'wifi': None,
'wifi_clients': None,
'bluetooth': None,
'rf': None,
'total_new': 0,
'total_missing': 0,
}
if wifi_devices is not None:
results['wifi'] = self.compare_wifi(wifi_devices)
results['total_new'] += results['wifi']['new_count']
results['total_missing'] += results['wifi']['missing_count']
if bt_devices is not None:
results['bluetooth'] = self.compare_bluetooth(bt_devices)
results['total_new'] += results['bluetooth']['new_count']
results['total_missing'] += results['bluetooth']['missing_count']
if wifi_devices is not None:
results['wifi'] = self.compare_wifi(wifi_devices)
results['total_new'] += results['wifi']['new_count']
results['total_missing'] += results['wifi']['missing_count']
if wifi_clients is not None:
results['wifi_clients'] = self.compare_wifi_clients(wifi_clients)
results['total_new'] += results['wifi_clients']['new_count']
results['total_missing'] += results['wifi_clients']['missing_count']
if bt_devices is not None:
results['bluetooth'] = self.compare_bluetooth(bt_devices)
results['total_new'] += results['bluetooth']['new_count']
results['total_missing'] += results['bluetooth']['missing_count']
if rf_signals is not None:
results['rf'] = self.compare_rf(rf_signals)
@@ -369,11 +445,12 @@ class BaselineComparator:
return results
def get_comparison_for_active_baseline(
wifi_devices: list[dict] | None = None,
bt_devices: list[dict] | None = None,
rf_signals: list[dict] | None = None
) -> dict | None:
def get_comparison_for_active_baseline(
wifi_devices: list[dict] | None = None,
wifi_clients: list[dict] | None = None,
bt_devices: list[dict] | None = None,
rf_signals: list[dict] | None = None
) -> dict | None:
"""
Convenience function to compare against the active baseline.
@@ -385,4 +462,4 @@ def get_comparison_for_active_baseline(
return None
comparator = BaselineComparator(baseline)
return comparator.compare_all(wifi_devices, bt_devices, rf_signals)
return comparator.compare_all(wifi_devices, wifi_clients, bt_devices, rf_signals)
utils/tscm/correlation.py
+439 -301
View File
@@ -22,7 +22,7 @@ logger = logging.getLogger('intercept.tscm.correlation')
class RiskLevel(Enum):
"""Risk classification levels."""
INFORMATIONAL = 'informational' # Score 0-2
NEEDS_REVIEW = 'review' # Score 3-5
NEEDS_REVIEW = 'needs_review' # Score 3-5
HIGH_INTEREST = 'high_interest' # Score 6+
@@ -118,10 +118,15 @@ class DeviceProfile:
identifier: str # MAC, BSSID, or frequency
protocol: str # 'bluetooth', 'wifi', 'rf'
# Device info
name: Optional[str] = None
manufacturer: Optional[str] = None
device_type: Optional[str] = None
# Device info
name: Optional[str] = None
manufacturer: Optional[str] = None
device_type: Optional[str] = None
tracker_type: Optional[str] = None
tracker_name: Optional[str] = None
tracker_confidence: Optional[str] = None
tracker_confidence_score: Optional[float] = None
tracker_evidence: list[str] = field(default_factory=list)
# Bluetooth-specific
services: list[str] = field(default_factory=list)
@@ -154,12 +159,12 @@ class DeviceProfile:
# Correlation
correlated_devices: list[str] = field(default_factory=list)
# Output
confidence: float = 0.0
recommended_action: str = 'monitor'
known_device: bool = False
known_device_name: Optional[str] = None
score_modifier: int = 0
# Output
confidence: float = 0.0
recommended_action: str = 'monitor'
known_device: bool = False
known_device_name: Optional[str] = None
score_modifier: int = 0
def add_rssi_sample(self, rssi: int) -> None:
"""Add an RSSI sample with timestamp."""
@@ -193,9 +198,9 @@ class DeviceProfile:
))
self._recalculate_score()
def _recalculate_score(self) -> None:
"""Recalculate total score and risk level."""
self.total_score = sum(i.score for i in self.indicators)
def _recalculate_score(self) -> None:
"""Recalculate total score and risk level."""
self.total_score = sum(i.score for i in self.indicators)
if self.total_score >= 6:
self.risk_level = RiskLevel.HIGH_INTEREST
@@ -207,38 +212,43 @@ class DeviceProfile:
self.risk_level = RiskLevel.INFORMATIONAL
self.recommended_action = 'monitor'
# Calculate confidence based on number and quality of indicators
indicator_count = len(self.indicators)
self.confidence = min(1.0, (indicator_count * 0.15) + (self.total_score * 0.05))
def apply_score_modifier(self, modifier: int | None) -> None:
"""Apply a score modifier (e.g., known-good device adjustment)."""
base_score = sum(i.score for i in self.indicators)
modifier_val = int(modifier) if modifier is not None else 0
self.score_modifier = modifier_val
self.total_score = max(0, base_score + modifier_val)
if self.total_score >= 6:
self.risk_level = RiskLevel.HIGH_INTEREST
self.recommended_action = 'investigate'
elif self.total_score >= 3:
self.risk_level = RiskLevel.NEEDS_REVIEW
self.recommended_action = 'review'
else:
self.risk_level = RiskLevel.INFORMATIONAL
self.recommended_action = 'monitor'
indicator_count = len(self.indicators)
self.confidence = min(1.0, (indicator_count * 0.15) + (self.total_score * 0.05))
# Calculate confidence based on number and quality of indicators
indicator_count = len(self.indicators)
self.confidence = min(1.0, (indicator_count * 0.15) + (self.total_score * 0.05))
def to_dict(self) -> dict:
"""Convert to dictionary for JSON serialization."""
return {
'identifier': self.identifier,
'protocol': self.protocol,
'name': self.name,
'manufacturer': self.manufacturer,
'device_type': self.device_type,
def apply_score_modifier(self, modifier: int | None) -> None:
"""Apply a score modifier (e.g., known-good device adjustment)."""
base_score = sum(i.score for i in self.indicators)
modifier_val = int(modifier) if modifier is not None else 0
self.score_modifier = modifier_val
self.total_score = max(0, base_score + modifier_val)
if self.total_score >= 6:
self.risk_level = RiskLevel.HIGH_INTEREST
self.recommended_action = 'investigate'
elif self.total_score >= 3:
self.risk_level = RiskLevel.NEEDS_REVIEW
self.recommended_action = 'review'
else:
self.risk_level = RiskLevel.INFORMATIONAL
self.recommended_action = 'monitor'
indicator_count = len(self.indicators)
self.confidence = min(1.0, (indicator_count * 0.15) + (self.total_score * 0.05))
def to_dict(self) -> dict:
"""Convert to dictionary for JSON serialization."""
return {
'identifier': self.identifier,
'protocol': self.protocol,
'name': self.name,
'manufacturer': self.manufacturer,
'device_type': self.device_type,
'tracker_type': self.tracker_type,
'tracker_name': self.tracker_name,
'tracker_confidence': self.tracker_confidence,
'tracker_confidence_score': self.tracker_confidence_score,
'tracker_evidence': self.tracker_evidence,
'ssid': self.ssid,
'frequency': self.frequency,
'first_seen': self.first_seen.isoformat() if self.first_seen else None,
@@ -254,26 +264,45 @@ class DeviceProfile:
}
for i in self.indicators
],
'total_score': self.total_score,
'score_modifier': self.score_modifier,
'risk_level': self.risk_level.value,
'confidence': round(self.confidence, 2),
'recommended_action': self.recommended_action,
'correlated_devices': self.correlated_devices,
'known_device': self.known_device,
'known_device_name': self.known_device_name,
}
'total_score': self.total_score,
'score_modifier': self.score_modifier,
'risk_level': self.risk_level.value,
'confidence': round(self.confidence, 2),
'recommended_action': self.recommended_action,
'correlated_devices': self.correlated_devices,
'known_device': self.known_device,
'known_device_name': self.known_device_name,
}
# Known audio-capable BLE service UUIDs
AUDIO_SERVICE_UUIDS = [
'0000110b-0000-1000-8000-00805f9b34fb', # A2DP Sink
'0000110a-0000-1000-8000-00805f9b34fb', # A2DP Source
'0000111e-0000-1000-8000-00805f9b34fb', # Handsfree
'0000111f-0000-1000-8000-00805f9b34fb', # Handsfree Audio Gateway
'00001108-0000-1000-8000-00805f9b34fb', # Headset
'00001203-0000-1000-8000-00805f9b34fb', # Generic Audio
]
AUDIO_SERVICE_UUIDS = [
'0000110b-0000-1000-8000-00805f9b34fb', # A2DP Sink
'0000110a-0000-1000-8000-00805f9b34fb', # A2DP Source
'0000111e-0000-1000-8000-00805f9b34fb', # Handsfree
'0000111f-0000-1000-8000-00805f9b34fb', # Handsfree Audio Gateway
'00001108-0000-1000-8000-00805f9b34fb', # Headset
'00001203-0000-1000-8000-00805f9b34fb', # Generic Audio
]
_BT_BASE_UUID_SUFFIX = '-0000-1000-8000-00805f9b34fb'
def _normalize_bt_uuid(value: str) -> str:
"""Normalize BLE UUIDs to 16-bit where possible."""
if not value:
return ''
uuid = str(value).lower().strip()
if uuid.startswith('0x'):
uuid = uuid[2:]
if uuid.endswith(_BT_BASE_UUID_SUFFIX) and len(uuid) >= 8:
return uuid[4:8]
if len(uuid) == 4:
return uuid
return uuid
AUDIO_SERVICE_UUIDS_16 = {_normalize_bt_uuid(u) for u in AUDIO_SERVICE_UUIDS}
# Generic chipset vendors (often used in covert devices)
GENERIC_CHIPSET_VENDORS = [
@@ -308,11 +337,11 @@ class CorrelationEngine:
potential surveillance activity patterns.
"""
def __init__(self):
self.device_profiles: dict[str, DeviceProfile] = {}
self.meeting_windows: list[tuple[datetime, datetime]] = []
self.correlation_window = timedelta(minutes=5)
self._known_device_cache: dict[str, dict | None] = {}
def __init__(self):
self.device_profiles: dict[str, DeviceProfile] = {}
self.meeting_windows: list[tuple[datetime, datetime]] = []
self.correlation_window = timedelta(minutes=5)
self._known_device_cache: dict[str, dict | None] = {}
def start_meeting_window(self) -> None:
"""Mark the start of a sensitive period (meeting)."""
@@ -326,64 +355,64 @@ class CorrelationEngine:
self.meeting_windows[-1] = (start, datetime.now())
logger.info("Meeting window ended")
def is_during_meeting(self, timestamp: datetime = None) -> bool:
"""Check if timestamp falls within a meeting window."""
ts = timestamp or datetime.now()
for start, end in self.meeting_windows:
if end is None:
if ts >= start:
return True
elif start <= ts <= end:
return True
return False
def _lookup_known_device(self, identifier: str, protocol: str) -> dict | None:
"""Lookup known-good device details with light normalization."""
cache_key = f"{protocol}:{identifier}"
if cache_key in self._known_device_cache:
return self._known_device_cache[cache_key]
try:
from utils.database import is_known_good_device
candidates = []
if identifier:
candidates.append(str(identifier))
if protocol == 'rf':
try:
freq_val = float(identifier)
candidates.append(f"{freq_val:.3f}")
candidates.append(f"{freq_val:.1f}")
except (ValueError, TypeError):
pass
known = None
for cand in candidates:
if not cand:
continue
known = is_known_good_device(str(cand).upper())
if known:
break
except Exception:
known = None
self._known_device_cache[cache_key] = known
return known
def _apply_known_device_modifier(self, profile: DeviceProfile, identifier: str, protocol: str) -> None:
"""Apply known-good score modifier and update profile metadata."""
known = self._lookup_known_device(identifier, protocol)
if known:
profile.known_device = True
profile.known_device_name = known.get('name') if isinstance(known, dict) else None
modifier = known.get('score_modifier', 0) if isinstance(known, dict) else 0
else:
profile.known_device = False
profile.known_device_name = None
modifier = 0
profile.apply_score_modifier(modifier)
def is_during_meeting(self, timestamp: datetime = None) -> bool:
"""Check if timestamp falls within a meeting window."""
ts = timestamp or datetime.now()
for start, end in self.meeting_windows:
if end is None:
if ts >= start:
return True
elif start <= ts <= end:
return True
return False
def _lookup_known_device(self, identifier: str, protocol: str) -> dict | None:
"""Lookup known-good device details with light normalization."""
cache_key = f"{protocol}:{identifier}"
if cache_key in self._known_device_cache:
return self._known_device_cache[cache_key]
try:
from utils.database import is_known_good_device
candidates = []
if identifier:
candidates.append(str(identifier))
if protocol == 'rf':
try:
freq_val = float(identifier)
candidates.append(f"{freq_val:.3f}")
candidates.append(f"{freq_val:.1f}")
except (ValueError, TypeError):
pass
known = None
for cand in candidates:
if not cand:
continue
known = is_known_good_device(str(cand).upper())
if known:
break
except Exception:
known = None
self._known_device_cache[cache_key] = known
return known
def _apply_known_device_modifier(self, profile: DeviceProfile, identifier: str, protocol: str) -> None:
"""Apply known-good score modifier and update profile metadata."""
known = self._lookup_known_device(identifier, protocol)
if known:
profile.known_device = True
profile.known_device_name = known.get('name') if isinstance(known, dict) else None
modifier = known.get('score_modifier', 0) if isinstance(known, dict) else 0
else:
profile.known_device = False
profile.known_device_name = None
modifier = 0
profile.apply_score_modifier(modifier)
def get_or_create_profile(self, identifier: str, protocol: str) -> DeviceProfile:
"""Get existing profile or create new one."""
@@ -415,10 +444,24 @@ class CorrelationEngine:
# Update profile data
profile.name = device.get('name') or profile.name
profile.manufacturer = device.get('manufacturer') or profile.manufacturer
profile.device_type = device.get('type') or profile.device_type
profile.services = device.get('services', []) or profile.services
profile.company_id = device.get('company_id') or profile.company_id
profile.advertising_interval = device.get('advertising_interval') or profile.advertising_interval
profile.device_type = device.get('type') or profile.device_type
services = device.get('services')
if not services:
services = device.get('service_uuids')
profile.services = services or profile.services
profile.company_id = device.get('company_id') or profile.company_id
profile.advertising_interval = device.get('advertising_interval') or profile.advertising_interval
tracker_data = device.get('tracker') or {}
if tracker_data:
profile.tracker_type = tracker_data.get('type') or profile.tracker_type
profile.tracker_name = tracker_data.get('name') or profile.tracker_name
profile.tracker_confidence = tracker_data.get('confidence') or profile.tracker_confidence
profile.tracker_confidence_score = tracker_data.get('confidence_score') or profile.tracker_confidence_score
evidence = tracker_data.get('evidence')
if isinstance(evidence, list):
profile.tracker_evidence = evidence
elif evidence:
profile.tracker_evidence = [str(evidence)]
# Add RSSI sample
rssi = device.get('rssi', device.get('signal'))
@@ -431,15 +474,28 @@ class CorrelationEngine:
# Clear previous indicators for fresh analysis
profile.indicators = []
# === Detection Logic ===
# 1. Unknown manufacturer or generic chipset
if not profile.manufacturer:
profile.add_indicator(
IndicatorType.UNKNOWN_DEVICE,
'Unknown manufacturer',
{'manufacturer': None}
)
# === Detection Logic ===
# 1. Unknown manufacturer or generic chipset
if not profile.manufacturer and mac and not device.get('is_randomized_mac'):
try:
first_octet = int(mac.split(':')[0], 16)
except (ValueError, IndexError):
first_octet = None
if first_octet is None or not (first_octet & 0x02):
try:
from data.oui import get_manufacturer
vendor = get_manufacturer(mac)
if vendor and vendor != 'Unknown':
profile.manufacturer = vendor
except Exception:
pass
if not profile.manufacturer:
profile.add_indicator(
IndicatorType.UNKNOWN_DEVICE,
'Unknown manufacturer',
{'manufacturer': None}
)
elif any(v in profile.manufacturer.lower() for v in GENERIC_CHIPSET_VENDORS):
profile.add_indicator(
IndicatorType.UNKNOWN_DEVICE,
@@ -455,16 +511,16 @@ class CorrelationEngine:
{'name': profile.name}
)
# 3. Audio-capable services
if profile.services:
audio_services = [s for s in profile.services
if s.lower() in [u.lower() for u in AUDIO_SERVICE_UUIDS]]
if audio_services:
profile.add_indicator(
IndicatorType.AUDIO_CAPABLE,
'Audio-capable BLE services detected',
{'services': audio_services}
)
# 3. Audio-capable services
if profile.services:
normalized_services = {_normalize_bt_uuid(s) for s in profile.services if s}
audio_services = [s for s in normalized_services if s in AUDIO_SERVICE_UUIDS_16]
if audio_services:
profile.add_indicator(
IndicatorType.AUDIO_CAPABLE,
'Audio-capable BLE services detected',
{'services': audio_services}
)
# Check name for audio keywords
if profile.name:
@@ -518,15 +574,47 @@ class CorrelationEngine:
{'mac': mac}
)
# 9. Known tracker detection (AirTag, Tile, SmartTag, ESP32)
mac_prefix = mac[:8] if len(mac) >= 8 else ''
tracker_detected = False
# Check for tracker flags from BLE scanner (manufacturer ID detection)
if device.get('is_airtag'):
profile.add_indicator(
IndicatorType.AIRTAG_DETECTED,
'Apple AirTag detected via manufacturer data',
# 9. Known tracker detection (AirTag, Tile, SmartTag, ESP32)
mac_prefix = mac[:8] if len(mac) >= 8 else ''
tracker_detected = False
tracker_data = device.get('tracker') or {}
if tracker_data.get('is_tracker'):
tracker_detected = True
tracker_label = tracker_data.get('name') or tracker_data.get('type')
if tracker_label:
label_lower = str(tracker_label).lower()
if 'airtag' in label_lower or 'find my' in label_lower:
profile.add_indicator(
IndicatorType.AIRTAG_DETECTED,
f'Tracker detected: {tracker_label}',
{'mac': mac, 'tracker_type': tracker_label}
)
profile.device_type = 'AirTag'
elif 'tile' in label_lower:
profile.add_indicator(
IndicatorType.TILE_DETECTED,
f'Tracker detected: {tracker_label}',
{'mac': mac, 'tracker_type': tracker_label}
)
profile.device_type = 'Tile Tracker'
elif 'smarttag' in label_lower or 'samsung' in label_lower:
profile.add_indicator(
IndicatorType.SMARTTAG_DETECTED,
f'Tracker detected: {tracker_label}',
{'mac': mac, 'tracker_type': tracker_label}
)
profile.device_type = 'Samsung SmartTag'
else:
profile.device_type = tracker_label
elif not profile.device_type:
profile.device_type = 'Tracker'
# Check for tracker flags from BLE scanner (manufacturer ID detection)
if device.get('is_airtag'):
profile.add_indicator(
IndicatorType.AIRTAG_DETECTED,
'Apple AirTag detected via manufacturer data',
{'mac': mac, 'tracker_type': 'AirTag'}
)
profile.device_type = device.get('tracker_type', 'AirTag')
@@ -634,59 +722,69 @@ class CorrelationEngine:
)
# Also check name for tracker keywords
if profile.name:
name_lower = profile.name.lower()
if 'airtag' in name_lower or 'findmy' in name_lower:
profile.add_indicator(
IndicatorType.AIRTAG_DETECTED,
f'AirTag identified by name: {profile.name}',
{'name': profile.name}
)
profile.device_type = 'AirTag'
elif 'tile' in name_lower:
profile.add_indicator(
IndicatorType.TILE_DETECTED,
f'Tile tracker identified by name: {profile.name}',
{'name': profile.name}
)
profile.device_type = 'Tile Tracker'
elif 'smarttag' in name_lower:
profile.add_indicator(
IndicatorType.SMARTTAG_DETECTED,
f'SmartTag identified by name: {profile.name}',
{'name': profile.name}
)
profile.device_type = 'Samsung SmartTag'
self._apply_known_device_modifier(profile, mac, 'bluetooth')
return profile
if profile.name:
name_lower = profile.name.lower()
if 'airtag' in name_lower or 'findmy' in name_lower:
profile.add_indicator(
IndicatorType.AIRTAG_DETECTED,
f'AirTag identified by name: {profile.name}',
{'name': profile.name}
)
profile.device_type = 'AirTag'
elif 'tile' in name_lower:
profile.add_indicator(
IndicatorType.TILE_DETECTED,
f'Tile tracker identified by name: {profile.name}',
{'name': profile.name}
)
profile.device_type = 'Tile Tracker'
elif 'smarttag' in name_lower:
profile.add_indicator(
IndicatorType.SMARTTAG_DETECTED,
f'SmartTag identified by name: {profile.name}',
{'name': profile.name}
)
profile.device_type = 'Samsung SmartTag'
def analyze_wifi_device(self, device: dict) -> DeviceProfile:
"""
Analyze a Wi-Fi device/AP for suspicious indicators.
self._apply_known_device_modifier(profile, mac, 'bluetooth')
return profile
def analyze_wifi_device(self, device: dict) -> DeviceProfile:
"""
Analyze a Wi-Fi device/AP for suspicious indicators.
Args:
device: Dict with bssid, ssid, channel, rssi, encryption, etc.
Returns:
DeviceProfile with risk assessment
"""
bssid = device.get('bssid', device.get('mac', '')).upper()
profile = self.get_or_create_profile(bssid, 'wifi')
# Update profile data
ssid = device.get('ssid', device.get('essid', ''))
profile.ssid = ssid if ssid else profile.ssid
profile.name = ssid or f'Hidden Network ({bssid[-8:]})'
profile.channel = device.get('channel') or profile.channel
profile.encryption = device.get('encryption', device.get('privacy')) or profile.encryption
profile.beacon_interval = device.get('beacon_interval') or profile.beacon_interval
profile.is_hidden = not ssid or ssid in ['', 'Hidden', '[Hidden]']
# Extract manufacturer from OUI
if bssid and len(bssid) >= 8:
profile.manufacturer = device.get('vendor') or profile.manufacturer
Returns:
DeviceProfile with risk assessment
"""
bssid = device.get('bssid', device.get('mac', '')).upper()
profile = self.get_or_create_profile(bssid, 'wifi')
is_client = bool(device.get('is_client') or device.get('role') == 'client')
# Update profile data
ssid = device.get('ssid', device.get('essid', ''))
if is_client:
profile.name = device.get('name') or device.get('vendor') or profile.name or f'Client ({bssid[-8:]})'
profile.device_type = 'client'
profile.ssid = profile.ssid # Clients are not SSIDs
profile.channel = device.get('channel') or profile.channel
profile.encryption = profile.encryption
profile.beacon_interval = profile.beacon_interval
profile.is_hidden = False
else:
profile.ssid = ssid if ssid else profile.ssid
profile.name = ssid or f'Hidden Network ({bssid[-8:]})'
profile.channel = device.get('channel') or profile.channel
profile.encryption = device.get('encryption', device.get('privacy')) or profile.encryption
profile.beacon_interval = device.get('beacon_interval') or profile.beacon_interval
profile.is_hidden = not ssid or ssid in ['', 'Hidden', '[Hidden]']
# Extract manufacturer from OUI
if bssid and len(bssid) >= 8:
profile.manufacturer = device.get('vendor') or profile.manufacturer
# Add RSSI sample
rssi = device.get('rssi', device.get('power', device.get('signal')))
@@ -699,82 +797,122 @@ class CorrelationEngine:
# Clear previous indicators
profile.indicators = []
# === Detection Logic ===
# 1. Hidden or unnamed SSID
if profile.is_hidden:
profile.add_indicator(
IndicatorType.HIDDEN_IDENTITY,
'Hidden or empty SSID',
{'ssid': ssid}
)
# 2. BSSID not in authorized list (would need baseline)
# For now, mark as unknown if no manufacturer
if not profile.manufacturer:
profile.add_indicator(
IndicatorType.UNKNOWN_DEVICE,
'Unknown AP manufacturer',
{'bssid': bssid}
)
# 3. Consumer device OUI in restricted environment
consumer_ouis = ['tp-link', 'netgear', 'd-link', 'linksys', 'asus']
if profile.manufacturer and any(c in profile.manufacturer.lower() for c in consumer_ouis):
profile.add_indicator(
IndicatorType.ROGUE_AP,
f'Consumer-grade AP detected: {profile.manufacturer}',
{'manufacturer': profile.manufacturer}
)
# 4. Camera device patterns
camera_keywords = ['cam', 'camera', 'ipcam', 'dvr', 'nvr', 'wyze',
'ring', 'arlo', 'nest', 'blink', 'eufy', 'yi']
if ssid and any(k in ssid.lower() for k in camera_keywords):
profile.add_indicator(
IndicatorType.AUDIO_CAPABLE, # Cameras often have mics
f'Potential camera device: {ssid}',
{'ssid': ssid}
)
# 5. Persistent presence
if profile.detection_count >= 3:
profile.add_indicator(
IndicatorType.PERSISTENT,
f'Persistent AP ({profile.detection_count} detections)',
{'count': profile.detection_count}
)
# 6. Stable RSSI (fixed placement)
rssi_stability = profile.get_rssi_stability()
if rssi_stability > 0.7 and len(profile.rssi_samples) >= 5:
profile.add_indicator(
IndicatorType.STABLE_RSSI,
f'Stable signal (stability: {rssi_stability:.0%})',
{'stability': rssi_stability}
)
# 7. Meeting correlation
if self.is_during_meeting():
profile.add_indicator(
IndicatorType.MEETING_CORRELATED,
'Detected during sensitive period',
{'during_meeting': True}
)
# 8. Strong hidden AP (very suspicious)
if profile.is_hidden and profile.rssi_samples:
latest_rssi = profile.rssi_samples[-1][1]
if latest_rssi > -50:
# === Detection Logic ===
if is_client:
if not profile.manufacturer:
profile.add_indicator(
IndicatorType.ROGUE_AP,
f'Strong hidden AP (RSSI: {latest_rssi} dBm)',
{'rssi': latest_rssi}
IndicatorType.UNKNOWN_DEVICE,
'Unknown client manufacturer',
{'mac': bssid}
)
self._apply_known_device_modifier(profile, bssid, 'wifi')
if profile.detection_count >= 3:
profile.add_indicator(
IndicatorType.PERSISTENT,
f'Persistent client ({profile.detection_count} detections)',
{'count': profile.detection_count}
)
return profile
rssi_stability = profile.get_rssi_stability()
if rssi_stability > 0.7 and len(profile.rssi_samples) >= 5:
profile.add_indicator(
IndicatorType.STABLE_RSSI,
f'Stable client signal (stability: {rssi_stability:.0%})',
{'stability': rssi_stability}
)
if self.is_during_meeting():
profile.add_indicator(
IndicatorType.MEETING_CORRELATED,
'Detected during sensitive period',
{'during_meeting': True}
)
try:
first_octet = int(bssid.split(':')[0], 16)
if first_octet & 0x02:
profile.add_indicator(
IndicatorType.MAC_ROTATION,
'Random/locally administered MAC detected',
{'mac': bssid}
)
except (ValueError, IndexError):
pass
else:
# 1. Hidden or unnamed SSID
if profile.is_hidden:
profile.add_indicator(
IndicatorType.HIDDEN_IDENTITY,
'Hidden or empty SSID',
{'ssid': ssid}
)
# 2. BSSID not in authorized list (would need baseline)
# For now, mark as unknown if no manufacturer
if not profile.manufacturer:
profile.add_indicator(
IndicatorType.UNKNOWN_DEVICE,
'Unknown AP manufacturer',
{'bssid': bssid}
)
# 3. Consumer device OUI in restricted environment
consumer_ouis = ['tp-link', 'netgear', 'd-link', 'linksys', 'asus']
if profile.manufacturer and any(c in profile.manufacturer.lower() for c in consumer_ouis):
profile.add_indicator(
IndicatorType.ROGUE_AP,
f'Consumer-grade AP detected: {profile.manufacturer}',
{'manufacturer': profile.manufacturer}
)
# 4. Camera device patterns
camera_keywords = ['cam', 'camera', 'ipcam', 'dvr', 'nvr', 'wyze',
'ring', 'arlo', 'nest', 'blink', 'eufy', 'yi']
if ssid and any(k in ssid.lower() for k in camera_keywords):
profile.add_indicator(
IndicatorType.AUDIO_CAPABLE, # Cameras often have mics
f'Potential camera device: {ssid}',
{'ssid': ssid}
)
# 5. Persistent presence
if profile.detection_count >= 3:
profile.add_indicator(
IndicatorType.PERSISTENT,
f'Persistent AP ({profile.detection_count} detections)',
{'count': profile.detection_count}
)
# 6. Stable RSSI (fixed placement)
rssi_stability = profile.get_rssi_stability()
if rssi_stability > 0.7 and len(profile.rssi_samples) >= 5:
profile.add_indicator(
IndicatorType.STABLE_RSSI,
f'Stable signal (stability: {rssi_stability:.0%})',
{'stability': rssi_stability}
)
# 7. Meeting correlation
if self.is_during_meeting():
profile.add_indicator(
IndicatorType.MEETING_CORRELATED,
'Detected during sensitive period',
{'during_meeting': True}
)
# 8. Strong hidden AP (very suspicious)
if profile.is_hidden and profile.rssi_samples:
latest_rssi = profile.rssi_samples[-1][1]
if latest_rssi > -50:
profile.add_indicator(
IndicatorType.ROGUE_AP,
f'Strong hidden AP (RSSI: {latest_rssi} dBm)',
{'rssi': latest_rssi}
)
self._apply_known_device_modifier(profile, bssid, 'wifi')
return profile
def analyze_rf_signal(self, signal: dict) -> DeviceProfile:
"""
@@ -857,16 +995,16 @@ class CorrelationEngine:
)
# 5. Meeting correlation
if self.is_during_meeting():
profile.add_indicator(
IndicatorType.MEETING_CORRELATED,
'Signal detected during sensitive period',
{'during_meeting': True}
)
self._apply_known_device_modifier(profile, freq_key, 'rf')
return profile
if self.is_during_meeting():
profile.add_indicator(
IndicatorType.MEETING_CORRELATED,
'Signal detected during sensitive period',
{'during_meeting': True}
)
self._apply_known_device_modifier(profile, freq_key, 'rf')
return profile
def correlate_devices(self) -> list[dict]:
"""
@@ -953,26 +1091,26 @@ class CorrelationEngine:
{'correlated_device': ap.identifier}
)
# Correlation 3: Same vendor BLE + WiFi
for bt in bt_devices:
if bt.manufacturer:
for wifi in wifi_devices:
if wifi.manufacturer and bt.manufacturer.lower() in wifi.manufacturer.lower():
correlation = {
# Correlation 3: Same vendor BLE + WiFi
for bt in bt_devices:
if bt.manufacturer:
for wifi in wifi_devices:
if wifi.manufacturer and bt.manufacturer.lower() in wifi.manufacturer.lower():
correlation = {
'type': 'same_vendor_bt_wifi',
'description': f'Same vendor ({bt.manufacturer}) on BLE and WiFi',
'devices': [bt.identifier, wifi.identifier],
'protocols': ['bluetooth', 'wifi'],
'score_boost': 2,
'significance': 'medium',
}
correlations.append(correlation)
# Re-apply known-good modifiers after correlation boosts
for profile in self.device_profiles.values():
self._apply_known_device_modifier(profile, profile.identifier, profile.protocol)
return correlations
}
correlations.append(correlation)
# Re-apply known-good modifiers after correlation boosts
for profile in self.device_profiles.values():
self._apply_known_device_modifier(profile, profile.identifier, profile.protocol)
return correlations
def get_high_interest_devices(self) -> list[DeviceProfile]:
"""Get all devices classified as high interest."""
utils/tscm/detector.py
+37 -19
View File
@@ -113,14 +113,18 @@ class ThreatDetector:
def _load_baseline(self, baseline: dict) -> None:
"""Load baseline device identifiers for comparison."""
# WiFi networks and clients
for network in baseline.get('wifi_networks', []):
if 'bssid' in network:
self.baseline_wifi_macs.add(network['bssid'].upper())
if 'clients' in network:
for client in network['clients']:
if 'mac' in client:
self.baseline_wifi_macs.add(client['mac'].upper())
# WiFi networks and clients
for network in baseline.get('wifi_networks', []):
if 'bssid' in network:
self.baseline_wifi_macs.add(network['bssid'].upper())
if 'clients' in network:
for client in network['clients']:
if 'mac' in client:
self.baseline_wifi_macs.add(client['mac'].upper())
for client in baseline.get('wifi_clients', []):
if 'mac' in client:
self.baseline_wifi_macs.add(client['mac'].upper())
# Bluetooth devices
for device in baseline.get('bt_devices', []):
@@ -476,11 +480,12 @@ class ThreatDetector:
mac = device.get('mac', device.get('address', '')).upper()
name = device.get('name', '')
rssi = device.get('rssi', device.get('signal', -100))
manufacturer = device.get('manufacturer', '')
device_type = device.get('type', '')
manufacturer_data = device.get('manufacturer_data')
threats = []
manufacturer = device.get('manufacturer', '')
device_type = device.get('type', '')
manufacturer_data = device.get('manufacturer_data')
tracker_data = device.get('tracker', {}) or {}
threats = []
# Check if new device (not in baseline)
if self.baseline and mac and mac not in self.baseline_bt_macs:
@@ -490,12 +495,25 @@ class ThreatDetector:
'reason': 'Device not present in baseline',
})
# Check for known trackers
tracker_info = is_known_tracker(name, manufacturer_data)
if tracker_info:
threats.append({
'type': 'tracker',
'severity': tracker_info.get('risk', 'high'),
# Check for known trackers (v2 tracker data if available)
if tracker_data.get('is_tracker'):
tracker_label = tracker_data.get('name') or tracker_data.get('type') or 'Tracker'
confidence = str(tracker_data.get('confidence') or '').lower()
severity = 'high' if confidence in ('high', 'medium') else 'medium'
threats.append({
'type': 'tracker',
'severity': severity,
'reason': f"Tracker detected: {tracker_label}",
'tracker_type': tracker_label,
'details': tracker_data.get('evidence', []),
})
# Check for known trackers (legacy patterns)
tracker_info = is_known_tracker(name, manufacturer_data)
if tracker_info:
threats.append({
'type': 'tracker',
'severity': tracker_info.get('risk', 'high'),
'reason': f"Known tracker detected: {tracker_info.get('name', 'Unknown')}",
'tracker_type': tracker_info.get('name'),
})
utils/tscm/reports.py
+59 -49
View File
@@ -102,13 +102,14 @@ class TSCMReport:
# Meeting window summaries
meeting_summaries: list[ReportMeetingSummary] = field(default_factory=list)
# Statistics
total_devices_scanned: int = 0
wifi_devices: int = 0
bluetooth_devices: int = 0
rf_signals: int = 0
new_devices: int = 0
missing_devices: int = 0
# Statistics
total_devices_scanned: int = 0
wifi_devices: int = 0
wifi_clients: int = 0
bluetooth_devices: int = 0
rf_signals: int = 0
new_devices: int = 0
missing_devices: int = 0
# Sweep duration
sweep_start: Optional[datetime] = None
@@ -200,12 +201,13 @@ def generate_executive_summary(report: TSCMReport) -> str:
lines.append("")
# Key statistics
lines.append("SCAN STATISTICS:")
lines.append(f" - Total devices scanned: {report.total_devices_scanned}")
lines.append(f" - WiFi access points: {report.wifi_devices}")
lines.append(f" - Bluetooth devices: {report.bluetooth_devices}")
lines.append(f" - RF signals: {report.rf_signals}")
lines.append("")
lines.append("SCAN STATISTICS:")
lines.append(f" - Total devices scanned: {report.total_devices_scanned}")
lines.append(f" - WiFi access points: {report.wifi_devices}")
lines.append(f" - WiFi clients: {report.wifi_clients}")
lines.append(f" - Bluetooth devices: {report.bluetooth_devices}")
lines.append(f" - RF signals: {report.rf_signals}")
lines.append("")
# Findings summary
lines.append("FINDINGS SUMMARY:")
@@ -427,13 +429,14 @@ def generate_technical_annex_json(report: TSCMReport) -> dict:
'capabilities': report.capabilities,
'limitations': report.limitations,
'statistics': {
'total_devices': report.total_devices_scanned,
'wifi_devices': report.wifi_devices,
'bluetooth_devices': report.bluetooth_devices,
'rf_signals': report.rf_signals,
'new_devices': report.new_devices,
'missing_devices': report.missing_devices,
'statistics': {
'total_devices': report.total_devices_scanned,
'wifi_devices': report.wifi_devices,
'wifi_clients': report.wifi_clients,
'bluetooth_devices': report.bluetooth_devices,
'rf_signals': report.rf_signals,
'new_devices': report.new_devices,
'missing_devices': report.missing_devices,
'high_interest_count': len(report.high_interest_findings),
'needs_review_count': len(report.needs_review_findings),
'informational_count': len(report.informational_findings),
@@ -781,21 +784,23 @@ class TSCMReportBuilder:
self.report.meeting_summaries.append(meeting)
return self
def add_statistics(
self,
wifi: int = 0,
bluetooth: int = 0,
rf: int = 0,
new: int = 0,
missing: int = 0
) -> 'TSCMReportBuilder':
self.report.wifi_devices = wifi
self.report.bluetooth_devices = bluetooth
self.report.rf_signals = rf
self.report.total_devices_scanned = wifi + bluetooth + rf
self.report.new_devices = new
self.report.missing_devices = missing
return self
def add_statistics(
self,
wifi: int = 0,
wifi_clients: int = 0,
bluetooth: int = 0,
rf: int = 0,
new: int = 0,
missing: int = 0
) -> 'TSCMReportBuilder':
self.report.wifi_devices = wifi
self.report.wifi_clients = wifi_clients
self.report.bluetooth_devices = bluetooth
self.report.rf_signals = rf
self.report.total_devices_scanned = wifi + wifi_clients + bluetooth + rf
self.report.new_devices = new
self.report.missing_devices = missing
return self
def add_device_timelines(self, timelines: list[dict]) -> 'TSCMReportBuilder':
self.report.device_timelines = timelines
@@ -890,25 +895,30 @@ def generate_report(
builder.add_findings_from_profiles(device_profiles)
# Statistics
results = sweep_data.get('results', {})
wifi_count = results.get('wifi_count')
if wifi_count is None:
wifi_count = len(results.get('wifi_devices', results.get('wifi', [])))
bt_count = results.get('bt_count')
if bt_count is None:
bt_count = len(results.get('bt_devices', results.get('bluetooth', [])))
results = sweep_data.get('results', {})
wifi_count = results.get('wifi_count')
if wifi_count is None:
wifi_count = len(results.get('wifi_devices', results.get('wifi', [])))
wifi_client_count = results.get('wifi_client_count')
if wifi_client_count is None:
wifi_client_count = len(results.get('wifi_clients', []))
bt_count = results.get('bt_count')
if bt_count is None:
bt_count = len(results.get('bt_devices', results.get('bluetooth', [])))
rf_count = results.get('rf_count')
if rf_count is None:
rf_count = len(results.get('rf_signals', results.get('rf', [])))
builder.add_statistics(
wifi=wifi_count,
bluetooth=bt_count,
rf=rf_count,
new=baseline_diff.get('summary', {}).get('new_devices', 0) if baseline_diff else 0,
missing=baseline_diff.get('summary', {}).get('missing_devices', 0) if baseline_diff else 0,
builder.add_statistics(
wifi=wifi_count,
wifi_clients=wifi_client_count,
bluetooth=bt_count,
rf=rf_count,
new=baseline_diff.get('summary', {}).get('new_devices', 0) if baseline_diff else 0,
missing=baseline_diff.get('summary', {}).get('missing_devices', 0) if baseline_diff else 0,
)
# Technical data
utils/wifi/constants.py
+21 -8
View File
@@ -414,14 +414,27 @@ VENDOR_OUIS = {
}
def get_vendor_from_mac(mac: str) -> str | None:
"""Get vendor name from MAC address OUI."""
if not mac:
return None
# Normalize MAC format
mac_upper = mac.upper().replace('-', ':')
oui = mac_upper[:8]
return VENDOR_OUIS.get(oui)
def get_vendor_from_mac(mac: str) -> str | None:
"""Get vendor name from MAC address OUI."""
if not mac:
return None
# Normalize MAC format
mac_upper = mac.upper().replace('-', ':')
oui = mac_upper[:8]
vendor = VENDOR_OUIS.get(oui)
if vendor:
return vendor
# Fallback to expanded OUI database if available
try:
from data.oui import get_manufacturer
manufacturer = get_manufacturer(mac_upper)
if manufacturer and manufacturer != 'Unknown':
return manufacturer
except Exception:
return None
return None
# =============================================================================
utils/wifi/models.py
+11 -10
View File
@@ -259,16 +259,17 @@ class WiFiAccessPoint:
'in_baseline': self.in_baseline,
}
def to_legacy_dict(self) -> dict:
"""Convert to legacy format for TSCM compatibility."""
return {
'bssid': self.bssid,
'essid': self.essid or '',
'power': str(self.rssi_current) if self.rssi_current else '-100',
'channel': str(self.channel) if self.channel else '',
'privacy': self.security,
'first_seen': self.first_seen.isoformat() if self.first_seen else '',
'last_seen': self.last_seen.isoformat() if self.last_seen else '',
def to_legacy_dict(self) -> dict:
"""Convert to legacy format for TSCM compatibility."""
return {
'bssid': self.bssid,
'essid': self.essid or '',
'vendor': self.vendor,
'power': str(self.rssi_current) if self.rssi_current else '-100',
'channel': str(self.channel) if self.channel else '',
'privacy': self.security,
'first_seen': self.first_seen.isoformat() if self.first_seen else '',
'last_seen': self.last_seen.isoformat() if self.last_seen else '',
'beacon_count': str(self.beacon_count),
'lan_ip': '', # Not tracked in new system
}
utils/wifi/scanner.py
+119 -19
View File
@@ -301,6 +301,73 @@ class UnifiedWiFiScanner:
return False
def _ensure_interface_up(self, interface: str) -> bool:
"""
Ensure a WiFi interface is up before scanning.
Attempts to bring the interface up using 'ip link set <iface> up',
falling back to 'ifconfig <iface> up'.
Args:
interface: Network interface name.
Returns:
True if the interface was brought up (or was already up),
False if we failed to bring it up.
"""
# Check current state via /sys/class/net
operstate_path = f"/sys/class/net/{interface}/operstate"
try:
with open(operstate_path) as f:
state = f.read().strip()
if state == "up":
return True
logger.info(f"Interface {interface} is '{state}', attempting to bring up")
except FileNotFoundError:
# Interface might not exist or /sys not available (non-Linux)
return True
except Exception:
pass
# Try ip link set up
if shutil.which('ip'):
try:
result = subprocess.run(
['ip', 'link', 'set', interface, 'up'],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0:
logger.info(f"Brought interface {interface} up via ip link")
time.sleep(1) # Brief settle time
return True
else:
logger.warning(f"ip link set {interface} up failed: {result.stderr.strip()}")
except Exception as e:
logger.warning(f"Failed to run ip link: {e}")
# Fallback to ifconfig
if shutil.which('ifconfig'):
try:
result = subprocess.run(
['ifconfig', interface, 'up'],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0:
logger.info(f"Brought interface {interface} up via ifconfig")
time.sleep(1)
return True
else:
logger.warning(f"ifconfig {interface} up failed: {result.stderr.strip()}")
except Exception as e:
logger.warning(f"Failed to run ifconfig: {e}")
logger.error(f"Could not bring interface {interface} up")
return False
# =========================================================================
# Quick Scan
# =========================================================================
@@ -362,6 +429,9 @@ class UnifiedWiFiScanner:
result.is_complete = True
return result
else: # Linux - try tools in order with fallback
# Ensure interface is up before scanning
self._ensure_interface_up(iface)
tools_to_try = []
if self._capabilities.has_nmcli:
tools_to_try.append(('nmcli', self._scan_with_nmcli))
@@ -375,6 +445,7 @@ class UnifiedWiFiScanner:
result.is_complete = True
return result
interface_was_down = False
for tool_name, scan_func in tools_to_try:
try:
logger.info(f"Attempting quick scan with {tool_name} on {iface}")
@@ -386,8 +457,28 @@ class UnifiedWiFiScanner:
error_msg = f"{tool_name}: {str(e)}"
errors_encountered.append(error_msg)
logger.warning(f"Quick scan with {tool_name} failed: {e}")
if 'is down' in str(e):
interface_was_down = True
continue # Try next tool
# If all tools failed because interface was down, try bringing it up and retry
if not tool_used and interface_was_down:
logger.info(f"Interface {iface} appears down, attempting to bring up and retry scan")
if self._ensure_interface_up(iface):
errors_encountered.clear()
for tool_name, scan_func in tools_to_try:
try:
logger.info(f"Retrying scan with {tool_name} on {iface} after bringing interface up")
observations = scan_func(iface, timeout)
tool_used = tool_name
logger.info(f"Retry scan with {tool_name} found {len(observations)} networks")
break
except Exception as e:
error_msg = f"{tool_name}: {str(e)}"
errors_encountered.append(error_msg)
logger.warning(f"Retry scan with {tool_name} failed: {e}")
continue
if not tool_used:
# All tools failed
result.error = "All scan tools failed. " + "; ".join(errors_encountered)
@@ -571,12 +662,13 @@ class UnifiedWiFiScanner:
# Deep Scan (airodump-ng)
# =========================================================================
def start_deep_scan(
self,
interface: Optional[str] = None,
band: str = 'all',
channel: Optional[int] = None,
) -> bool:
def start_deep_scan(
self,
interface: Optional[str] = None,
band: str = 'all',
channel: Optional[int] = None,
channels: Optional[list[int]] = None,
) -> bool:
"""
Start continuous deep scan with airodump-ng.
@@ -609,11 +701,11 @@ class UnifiedWiFiScanner:
# Start airodump-ng in background thread
self._deep_scan_stop_event.clear()
self._deep_scan_thread = threading.Thread(
target=self._run_deep_scan,
args=(iface, band, channel),
daemon=True,
)
self._deep_scan_thread = threading.Thread(
target=self._run_deep_scan,
args=(iface, band, channel, channels),
daemon=True,
)
self._deep_scan_thread.start()
self._status = WiFiScanStatus(
@@ -675,8 +767,14 @@ class UnifiedWiFiScanner:
return True
def _run_deep_scan(self, interface: str, band: str, channel: Optional[int]):
"""Background thread for running airodump-ng."""
def _run_deep_scan(
self,
interface: str,
band: str,
channel: Optional[int],
channels: Optional[list[int]],
):
"""Background thread for running airodump-ng."""
from .parsers.airodump import parse_airodump_csv
import tempfile
@@ -688,12 +786,14 @@ class UnifiedWiFiScanner:
# Build command
cmd = ['airodump-ng', '-w', output_prefix, '--output-format', 'csv']
if channel:
cmd.extend(['-c', str(channel)])
elif band == '2.4':
cmd.extend(['--band', 'bg'])
elif band == '5':
cmd.extend(['--band', 'a'])
if channels:
cmd.extend(['-c', ','.join(str(c) for c in channels)])
elif channel:
cmd.extend(['-c', str(channel)])
elif band == '2.4':
cmd.extend(['--band', 'bg'])
elif band == '5':
cmd.extend(['--band', 'a'])
cmd.append(interface)