Add disk space monitoring to recording lifecycle

This commit is contained in:
Ember
2026-02-10 16:28:21 -08:00
committed by Cooper Quintin
parent ec6967e2a1
commit 0c624c2bc2
7 changed files with 244 additions and 41 deletions

View File

@@ -20,6 +20,20 @@ pub struct Config {
pub ntfy_url: Option<String>,
pub enabled_notifications: Vec<NotificationType>,
pub analyzers: AnalyzerConfig,
/// Minimum disk space (MB) required to start recording
#[serde(default = "default_min_space_to_start_recording_mb")]
pub min_space_to_start_recording_mb: u64,
/// Minimum disk space (MB) to continue recording (stops if below this)
#[serde(default = "default_min_space_to_continue_recording_mb")]
pub min_space_to_continue_recording_mb: u64,
}
fn default_min_space_to_start_recording_mb() -> u64 {
1
}
fn default_min_space_to_continue_recording_mb() -> u64 {
1
}
impl Default for Config {
@@ -35,6 +49,8 @@ impl Default for Config {
analyzers: AnalyzerConfig::default(),
ntfy_url: None,
enabled_notifications: vec![NotificationType::Warning, NotificationType::LowBattery],
min_space_to_start_recording_mb: default_min_space_to_start_recording_mb(),
min_space_to_continue_recording_mb: default_min_space_to_continue_recording_mb(),
}
}
}

View File

@@ -27,6 +27,9 @@ use crate::display;
use crate::notifications::{Notification, NotificationType};
use crate::qmdl_store::{RecordingStore, RecordingStoreError};
use crate::server::ServerState;
use crate::stats::DiskStats;
const SPACE_CHECK_INTERVAL_CONTAINERS: usize = 100;
pub enum DiagDeviceCtrlMessage {
StopRecording,
@@ -46,8 +49,11 @@ pub struct DiagTask {
analysis_sender: Sender<AnalysisCtrlMessage>,
analyzer_config: AnalyzerConfig,
notification_channel: tokio::sync::mpsc::Sender<Notification>,
min_space_to_start_mb: u64,
min_space_to_continue_mb: u64,
state: DiagState,
max_type_seen: EventType,
container_count: usize,
}
enum DiagState {
@@ -64,30 +70,90 @@ impl DiagTask {
analysis_sender: Sender<AnalysisCtrlMessage>,
analyzer_config: AnalyzerConfig,
notification_channel: tokio::sync::mpsc::Sender<Notification>,
min_space_to_start_mb: u64,
min_space_to_continue_mb: u64,
) -> Self {
Self {
ui_update_sender,
analysis_sender,
analyzer_config,
notification_channel,
min_space_to_start_mb,
min_space_to_continue_mb,
state: DiagState::Stopped,
max_type_seen: EventType::Informational,
container_count: 0,
}
}
/// Start recording
async fn start(&mut self, qmdl_store: &mut RecordingStore) {
self.max_type_seen = EventType::Informational;
let (qmdl_file, analysis_file) = qmdl_store
.new_entry()
.await
.expect("failed creating QMDL file entry");
self.container_count = 0;
let min_space_bytes = self.min_space_to_start_mb * 1024 * 1024;
match DiskStats::new(qmdl_store.path.to_str().unwrap()) {
Ok(disk_stats) if disk_stats.available_bytes.unwrap_or(0) < min_space_bytes => {
let available_mb = disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
error!(
"Insufficient disk space to start recording: {}MB available, {}MB required",
available_mb, self.min_space_to_start_mb
);
if let Err(e) = self
.notification_channel
.send(Notification::new(
NotificationType::Warning,
format!(
"Cannot start recording: only {}MB free (need {}MB minimum)",
available_mb, self.min_space_to_start_mb
),
None,
))
.await
{
warn!("Failed to send notification: {e}");
}
if let Err(e) = self
.ui_update_sender
.send(display::DisplayState::Paused)
.await
{
warn!("couldn't send ui update message: {e}");
}
return;
}
Ok(disk_stats) => {
let available_mb = disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
info!(
"Starting recording with {}MB disk space available",
available_mb
);
}
Err(e) => {
warn!("Failed to check disk space: {e}, starting recording anyway");
}
}
let (qmdl_file, analysis_file) = match qmdl_store.new_entry().await {
Ok(files) => files,
Err(e) => {
error!("failed creating QMDL file entry: {e}");
return;
}
};
self.stop_current_recording().await;
let qmdl_writer = QmdlWriter::new(qmdl_file);
let analysis_writer = AnalysisWriter::new(analysis_file, &self.analyzer_config)
.await
.map(Box::new)
.expect("failed to write to analysis file");
let analysis_writer = match AnalysisWriter::new(analysis_file, &self.analyzer_config).await
{
Ok(writer) => Box::new(writer),
Err(e) => {
error!("failed to create analysis writer: {e}");
return;
}
};
self.state = DiagState::Recording {
qmdl_writer,
analysis_writer,
@@ -183,10 +249,60 @@ impl DiagTask {
analysis_writer,
} = &mut self.state
{
qmdl_writer
.write_container(&container)
.await
.expect("failed to write to QMDL writer");
self.container_count += 1;
if self.container_count % SPACE_CHECK_INTERVAL_CONTAINERS == 0 {
let min_continue_bytes = self.min_space_to_continue_mb * 1024 * 1024;
let min_start_bytes = self.min_space_to_start_mb * 1024 * 1024;
match DiskStats::new(qmdl_store.path.to_str().unwrap()) {
Ok(disk_stats)
if disk_stats.available_bytes.unwrap_or(0) < min_continue_bytes =>
{
let available_mb = disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
error!(
"Disk space critically low ({}MB), stopping recording",
available_mb
);
self.notification_channel.send(Notification::new(
NotificationType::Warning,
format!(
"Disk space critically low ({}MB), recording stopped automatically",
available_mb
),
None,
)).await.ok();
self.stop(qmdl_store).await;
return;
}
Ok(disk_stats) if disk_stats.available_bytes.unwrap_or(0) < min_start_bytes => {
if self.container_count % (SPACE_CHECK_INTERVAL_CONTAINERS * 10) == 0 {
let available_mb =
disk_stats.available_bytes.unwrap_or(0) / 1024 / 1024;
warn!("Disk space low: {}MB remaining", available_mb);
self.notification_channel
.send(Notification::new(
NotificationType::Warning,
format!("Disk space low: {}MB free", available_mb),
Some(Duration::from_secs(30)),
))
.await
.ok();
}
}
Err(e) => {
warn!("Failed to check disk space: {e}");
}
_ => {}
}
}
if let Err(e) = qmdl_writer.write_container(&container).await {
error!("failed to write to QMDL (disk full?): {e}");
self.stop(qmdl_store).await;
return;
}
debug!(
"total QMDL bytes written: {}, updating manifest...",
qmdl_writer.total_written
@@ -194,15 +310,22 @@ impl DiagTask {
let index = qmdl_store
.current_entry
.expect("DiagDevice had qmdl_writer, but QmdlStore didn't have current entry???");
qmdl_store
if let Err(e) = qmdl_store
.update_entry_qmdl_size(index, qmdl_writer.total_written)
.await
.expect("failed to update qmdl file size");
{
error!("failed to update manifest (disk full?): {e}");
self.stop(qmdl_store).await;
return;
}
debug!("done!");
let max_type = analysis_writer
.analyze(container)
.await
.expect("failed to analyze container");
let max_type = match analysis_writer.analyze(container).await {
Ok(t) => t,
Err(e) => {
warn!("failed to analyze container: {e}");
EventType::Informational
}
};
if max_type > EventType::Informational {
info!("a heuristic triggered on this run!");
@@ -244,10 +367,12 @@ pub fn run_diag_read_thread(
analysis_sender: Sender<AnalysisCtrlMessage>,
analyzer_config: AnalyzerConfig,
notification_channel: tokio::sync::mpsc::Sender<Notification>,
min_space_to_start_mb: u64,
min_space_to_continue_mb: u64,
) {
task_tracker.spawn(async move {
let mut diag_stream = pin!(dev.as_stream().into_stream());
let mut diag_task = DiagTask::new(ui_update_sender, analysis_sender, analyzer_config, notification_channel);
let mut diag_task = DiagTask::new(ui_update_sender, analysis_sender, analyzer_config, notification_channel, min_space_to_start_mb, min_space_to_continue_mb);
qmdl_file_tx
.send(DiagDeviceCtrlMessage::StartRecording)
.await

View File

@@ -234,6 +234,8 @@ async fn run_with_config(
analysis_tx.clone(),
config.analyzers.clone(),
notification_service.new_handler(),
config.min_space_to_start_recording_mb,
config.min_space_to_continue_recording_mb,
);
info!("Starting UI");

View File

@@ -1,3 +1,4 @@
use std::ffi::CString;
use std::sync::Arc;
use crate::battery::get_battery_status;
@@ -25,7 +26,7 @@ pub struct SystemStats {
impl SystemStats {
pub async fn new(qmdl_path: &str, device: &Device) -> Result<Self, String> {
Ok(Self {
disk_stats: DiskStats::new(qmdl_path, device).await?,
disk_stats: DiskStats::new(qmdl_path)?,
memory_stats: MemoryStats::new(device).await?,
runtime_metadata: RuntimeMetadata::new(),
battery_status: match get_battery_status(device).await {
@@ -48,33 +49,42 @@ pub struct DiskStats {
available_size: String,
used_percent: String,
mounted_on: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub available_bytes: Option<u64>,
}
impl DiskStats {
// runs "df -h <qmdl_path>" to get storage statistics for the partition containing
// the QMDL file.
pub async fn new(qmdl_path: &str, device: &Device) -> Result<Self, String> {
// Uz801 needs to be told to use the busybox df specifically
let mut df_cmd: Command;
if matches!(device, Device::Uz801) {
df_cmd = Command::new("busybox");
df_cmd.arg("df");
} else {
df_cmd = Command::new("df");
#[allow(clippy::unnecessary_cast)] // c_ulong is u32 on ARM, u64 on macOS
pub fn new(qmdl_path: &str) -> Result<Self, String> {
let c_path =
CString::new(qmdl_path).map_err(|e| format!("invalid path {qmdl_path}: {e}"))?;
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
if unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) } != 0 {
return Err(format!(
"statvfs({qmdl_path}) failed: {}",
std::io::Error::last_os_error()
));
}
df_cmd.arg("-h");
df_cmd.arg(qmdl_path);
let stdout = get_cmd_output(df_cmd).await?;
// Handle standard df -h format
let mut parts = stdout.split_whitespace().skip(7);
let block_size = stat.f_frsize as u64;
let total_kb = (stat.f_blocks as u64 * block_size / 1024) as usize;
let free_kb = (stat.f_bfree as u64 * block_size / 1024) as usize;
let available_kb = (stat.f_bavail as u64 * block_size / 1024) as usize;
let used_kb = total_kb.saturating_sub(free_kb);
let used_percent = if stat.f_blocks > 0 {
format!("{}%", (stat.f_blocks - stat.f_bfree) * 100 / stat.f_blocks)
} else {
"0%".to_string()
};
Ok(Self {
partition: parts.next().ok_or("error parsing df output")?.to_string(),
total_size: parts.next().ok_or("error parsing df output")?.to_string(),
used_size: parts.next().ok_or("error parsing df output")?.to_string(),
available_size: parts.next().ok_or("error parsing df output")?.to_string(),
used_percent: parts.next().ok_or("error parsing df output")?.to_string(),
mounted_on: parts.next().ok_or("error parsing df output")?.to_string(),
partition: qmdl_path.to_string(),
total_size: humanize_kb(total_kb),
used_size: humanize_kb(used_kb),
available_size: humanize_kb(available_kb),
used_percent,
mounted_on: qmdl_path.to_string(),
available_bytes: Some(stat.f_bavail as u64 * block_size),
})
}
}