mirror of
https://github.com/EFForg/rayhunter.git
synced 2026-04-27 07:59:59 -07:00
Add disk space monitoring to recording lifecycle
This commit is contained in:
@@ -20,6 +20,20 @@ pub struct Config {
|
||||
pub ntfy_url: Option<String>,
|
||||
pub enabled_notifications: Vec<NotificationType>,
|
||||
pub analyzers: AnalyzerConfig,
|
||||
/// Minimum disk space (MB) required to start recording
|
||||
#[serde(default = "default_min_space_to_start_recording_mb")]
|
||||
pub min_space_to_start_recording_mb: u64,
|
||||
/// Minimum disk space (MB) to continue recording (stops if below this)
|
||||
#[serde(default = "default_min_space_to_continue_recording_mb")]
|
||||
pub min_space_to_continue_recording_mb: u64,
|
||||
}
|
||||
|
||||
fn default_min_space_to_start_recording_mb() -> u64 {
|
||||
1
|
||||
}
|
||||
|
||||
fn default_min_space_to_continue_recording_mb() -> u64 {
|
||||
1
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
@@ -35,6 +49,8 @@ impl Default for Config {
|
||||
analyzers: AnalyzerConfig::default(),
|
||||
ntfy_url: None,
|
||||
enabled_notifications: vec![NotificationType::Warning, NotificationType::LowBattery],
|
||||
min_space_to_start_recording_mb: default_min_space_to_start_recording_mb(),
|
||||
min_space_to_continue_recording_mb: default_min_space_to_continue_recording_mb(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,6 +27,9 @@ use crate::display;
|
||||
use crate::notifications::{Notification, NotificationType};
|
||||
use crate::qmdl_store::{RecordingStore, RecordingStoreError};
|
||||
use crate::server::ServerState;
|
||||
use crate::stats::DiskStats;
|
||||
|
||||
const SPACE_CHECK_INTERVAL_CONTAINERS: usize = 100;
|
||||
|
||||
pub enum DiagDeviceCtrlMessage {
|
||||
StopRecording,
|
||||
@@ -46,8 +49,11 @@ pub struct DiagTask {
|
||||
analysis_sender: Sender<AnalysisCtrlMessage>,
|
||||
analyzer_config: AnalyzerConfig,
|
||||
notification_channel: tokio::sync::mpsc::Sender<Notification>,
|
||||
min_space_to_start_mb: u64,
|
||||
min_space_to_continue_mb: u64,
|
||||
state: DiagState,
|
||||
max_type_seen: EventType,
|
||||
container_count: usize,
|
||||
}
|
||||
|
||||
enum DiagState {
|
||||
@@ -64,30 +70,90 @@ impl DiagTask {
|
||||
analysis_sender: Sender<AnalysisCtrlMessage>,
|
||||
analyzer_config: AnalyzerConfig,
|
||||
notification_channel: tokio::sync::mpsc::Sender<Notification>,
|
||||
min_space_to_start_mb: u64,
|
||||
min_space_to_continue_mb: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
ui_update_sender,
|
||||
analysis_sender,
|
||||
analyzer_config,
|
||||
notification_channel,
|
||||
min_space_to_start_mb,
|
||||
min_space_to_continue_mb,
|
||||
state: DiagState::Stopped,
|
||||
max_type_seen: EventType::Informational,
|
||||
container_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start recording
|
||||
async fn start(&mut self, qmdl_store: &mut RecordingStore) {
|
||||
self.max_type_seen = EventType::Informational;
|
||||
let (qmdl_file, analysis_file) = qmdl_store
|
||||
.new_entry()
|
||||
.await
|
||||
.expect("failed creating QMDL file entry");
|
||||
self.container_count = 0;
|
||||
|
||||
let min_space_bytes = self.min_space_to_start_mb * 1024 * 1024;
|
||||
match DiskStats::new(qmdl_store.path.to_str().unwrap()) {
|
||||
Ok(disk_stats) if disk_stats.available_bytes.unwrap_or(0) < min_space_bytes => {
|
||||
let available_mb = disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
|
||||
error!(
|
||||
"Insufficient disk space to start recording: {}MB available, {}MB required",
|
||||
available_mb, self.min_space_to_start_mb
|
||||
);
|
||||
|
||||
if let Err(e) = self
|
||||
.notification_channel
|
||||
.send(Notification::new(
|
||||
NotificationType::Warning,
|
||||
format!(
|
||||
"Cannot start recording: only {}MB free (need {}MB minimum)",
|
||||
available_mb, self.min_space_to_start_mb
|
||||
),
|
||||
None,
|
||||
))
|
||||
.await
|
||||
{
|
||||
warn!("Failed to send notification: {e}");
|
||||
}
|
||||
|
||||
if let Err(e) = self
|
||||
.ui_update_sender
|
||||
.send(display::DisplayState::Paused)
|
||||
.await
|
||||
{
|
||||
warn!("couldn't send ui update message: {e}");
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
Ok(disk_stats) => {
|
||||
let available_mb = disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
|
||||
info!(
|
||||
"Starting recording with {}MB disk space available",
|
||||
available_mb
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to check disk space: {e}, starting recording anyway");
|
||||
}
|
||||
}
|
||||
|
||||
let (qmdl_file, analysis_file) = match qmdl_store.new_entry().await {
|
||||
Ok(files) => files,
|
||||
Err(e) => {
|
||||
error!("failed creating QMDL file entry: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.stop_current_recording().await;
|
||||
let qmdl_writer = QmdlWriter::new(qmdl_file);
|
||||
let analysis_writer = AnalysisWriter::new(analysis_file, &self.analyzer_config)
|
||||
.await
|
||||
.map(Box::new)
|
||||
.expect("failed to write to analysis file");
|
||||
let analysis_writer = match AnalysisWriter::new(analysis_file, &self.analyzer_config).await
|
||||
{
|
||||
Ok(writer) => Box::new(writer),
|
||||
Err(e) => {
|
||||
error!("failed to create analysis writer: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
self.state = DiagState::Recording {
|
||||
qmdl_writer,
|
||||
analysis_writer,
|
||||
@@ -183,10 +249,60 @@ impl DiagTask {
|
||||
analysis_writer,
|
||||
} = &mut self.state
|
||||
{
|
||||
qmdl_writer
|
||||
.write_container(&container)
|
||||
.await
|
||||
.expect("failed to write to QMDL writer");
|
||||
self.container_count += 1;
|
||||
|
||||
if self.container_count % SPACE_CHECK_INTERVAL_CONTAINERS == 0 {
|
||||
let min_continue_bytes = self.min_space_to_continue_mb * 1024 * 1024;
|
||||
let min_start_bytes = self.min_space_to_start_mb * 1024 * 1024;
|
||||
match DiskStats::new(qmdl_store.path.to_str().unwrap()) {
|
||||
Ok(disk_stats)
|
||||
if disk_stats.available_bytes.unwrap_or(0) < min_continue_bytes =>
|
||||
{
|
||||
let available_mb = disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
|
||||
error!(
|
||||
"Disk space critically low ({}MB), stopping recording",
|
||||
available_mb
|
||||
);
|
||||
|
||||
self.notification_channel.send(Notification::new(
|
||||
NotificationType::Warning,
|
||||
format!(
|
||||
"Disk space critically low ({}MB), recording stopped automatically",
|
||||
available_mb
|
||||
),
|
||||
None,
|
||||
)).await.ok();
|
||||
|
||||
self.stop(qmdl_store).await;
|
||||
return;
|
||||
}
|
||||
Ok(disk_stats) if disk_stats.available_bytes.unwrap_or(0) < min_start_bytes => {
|
||||
if self.container_count % (SPACE_CHECK_INTERVAL_CONTAINERS * 10) == 0 {
|
||||
let available_mb =
|
||||
disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
|
||||
warn!("Disk space low: {}MB remaining", available_mb);
|
||||
self.notification_channel
|
||||
.send(Notification::new(
|
||||
NotificationType::Warning,
|
||||
format!("Disk space low: {}MB free", available_mb),
|
||||
Some(Duration::from_secs(30)),
|
||||
))
|
||||
.await
|
||||
.ok();
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to check disk space: {e}");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(e) = qmdl_writer.write_container(&container).await {
|
||||
error!("failed to write to QMDL (disk full?): {e}");
|
||||
self.stop(qmdl_store).await;
|
||||
return;
|
||||
}
|
||||
debug!(
|
||||
"total QMDL bytes written: {}, updating manifest...",
|
||||
qmdl_writer.total_written
|
||||
@@ -194,15 +310,22 @@ impl DiagTask {
|
||||
let index = qmdl_store
|
||||
.current_entry
|
||||
.expect("DiagDevice had qmdl_writer, but QmdlStore didn't have current entry???");
|
||||
qmdl_store
|
||||
if let Err(e) = qmdl_store
|
||||
.update_entry_qmdl_size(index, qmdl_writer.total_written)
|
||||
.await
|
||||
.expect("failed to update qmdl file size");
|
||||
{
|
||||
error!("failed to update manifest (disk full?): {e}");
|
||||
self.stop(qmdl_store).await;
|
||||
return;
|
||||
}
|
||||
debug!("done!");
|
||||
let max_type = analysis_writer
|
||||
.analyze(container)
|
||||
.await
|
||||
.expect("failed to analyze container");
|
||||
let max_type = match analysis_writer.analyze(container).await {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
warn!("failed to analyze container: {e}");
|
||||
EventType::Informational
|
||||
}
|
||||
};
|
||||
|
||||
if max_type > EventType::Informational {
|
||||
info!("a heuristic triggered on this run!");
|
||||
@@ -244,10 +367,12 @@ pub fn run_diag_read_thread(
|
||||
analysis_sender: Sender<AnalysisCtrlMessage>,
|
||||
analyzer_config: AnalyzerConfig,
|
||||
notification_channel: tokio::sync::mpsc::Sender<Notification>,
|
||||
min_space_to_start_mb: u64,
|
||||
min_space_to_continue_mb: u64,
|
||||
) {
|
||||
task_tracker.spawn(async move {
|
||||
let mut diag_stream = pin!(dev.as_stream().into_stream());
|
||||
let mut diag_task = DiagTask::new(ui_update_sender, analysis_sender, analyzer_config, notification_channel);
|
||||
let mut diag_task = DiagTask::new(ui_update_sender, analysis_sender, analyzer_config, notification_channel, min_space_to_start_mb, min_space_to_continue_mb);
|
||||
qmdl_file_tx
|
||||
.send(DiagDeviceCtrlMessage::StartRecording)
|
||||
.await
|
||||
|
||||
@@ -234,6 +234,8 @@ async fn run_with_config(
|
||||
analysis_tx.clone(),
|
||||
config.analyzers.clone(),
|
||||
notification_service.new_handler(),
|
||||
config.min_space_to_start_recording_mb,
|
||||
config.min_space_to_continue_recording_mb,
|
||||
);
|
||||
info!("Starting UI");
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::ffi::CString;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::battery::get_battery_status;
|
||||
@@ -25,7 +26,7 @@ pub struct SystemStats {
|
||||
impl SystemStats {
|
||||
pub async fn new(qmdl_path: &str, device: &Device) -> Result<Self, String> {
|
||||
Ok(Self {
|
||||
disk_stats: DiskStats::new(qmdl_path, device).await?,
|
||||
disk_stats: DiskStats::new(qmdl_path)?,
|
||||
memory_stats: MemoryStats::new(device).await?,
|
||||
runtime_metadata: RuntimeMetadata::new(),
|
||||
battery_status: match get_battery_status(device).await {
|
||||
@@ -48,33 +49,42 @@ pub struct DiskStats {
|
||||
available_size: String,
|
||||
used_percent: String,
|
||||
mounted_on: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub available_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
impl DiskStats {
|
||||
// runs "df -h <qmdl_path>" to get storage statistics for the partition containing
|
||||
// the QMDL file.
|
||||
pub async fn new(qmdl_path: &str, device: &Device) -> Result<Self, String> {
|
||||
// Uz801 needs to be told to use the busybox df specifically
|
||||
let mut df_cmd: Command;
|
||||
if matches!(device, Device::Uz801) {
|
||||
df_cmd = Command::new("busybox");
|
||||
df_cmd.arg("df");
|
||||
} else {
|
||||
df_cmd = Command::new("df");
|
||||
#[allow(clippy::unnecessary_cast)] // c_ulong is u32 on ARM, u64 on macOS
|
||||
pub fn new(qmdl_path: &str) -> Result<Self, String> {
|
||||
let c_path =
|
||||
CString::new(qmdl_path).map_err(|e| format!("invalid path {qmdl_path}: {e}"))?;
|
||||
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
|
||||
if unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) } != 0 {
|
||||
return Err(format!(
|
||||
"statvfs({qmdl_path}) failed: {}",
|
||||
std::io::Error::last_os_error()
|
||||
));
|
||||
}
|
||||
df_cmd.arg("-h");
|
||||
df_cmd.arg(qmdl_path);
|
||||
let stdout = get_cmd_output(df_cmd).await?;
|
||||
|
||||
// Handle standard df -h format
|
||||
let mut parts = stdout.split_whitespace().skip(7);
|
||||
let block_size = stat.f_frsize as u64;
|
||||
let total_kb = (stat.f_blocks as u64 * block_size / 1024) as usize;
|
||||
let free_kb = (stat.f_bfree as u64 * block_size / 1024) as usize;
|
||||
let available_kb = (stat.f_bavail as u64 * block_size / 1024) as usize;
|
||||
let used_kb = total_kb.saturating_sub(free_kb);
|
||||
let used_percent = if stat.f_blocks > 0 {
|
||||
format!("{}%", (stat.f_blocks - stat.f_bfree) * 100 / stat.f_blocks)
|
||||
} else {
|
||||
"0%".to_string()
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
partition: parts.next().ok_or("error parsing df output")?.to_string(),
|
||||
total_size: parts.next().ok_or("error parsing df output")?.to_string(),
|
||||
used_size: parts.next().ok_or("error parsing df output")?.to_string(),
|
||||
available_size: parts.next().ok_or("error parsing df output")?.to_string(),
|
||||
used_percent: parts.next().ok_or("error parsing df output")?.to_string(),
|
||||
mounted_on: parts.next().ok_or("error parsing df output")?.to_string(),
|
||||
partition: qmdl_path.to_string(),
|
||||
total_size: humanize_kb(total_kb),
|
||||
used_size: humanize_kb(used_kb),
|
||||
available_size: humanize_kb(available_kb),
|
||||
used_percent,
|
||||
mounted_on: qmdl_path.to_string(),
|
||||
available_bytes: Some(stat.f_bavail as u64 * block_size),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user