Add support for WebDAV (#982)

* Add support for WebDAV

* Fix get_unuploaded_entries_with_age to use start_time

* Use better pattern when matching join! result

Co-authored-by: Markus Unterwaditzer <markus-github@unterwaditzer.net>

* Refactor Webdav with better string ownership

* Unformat Cargo.toml

* Add timeout config

* Use a single url config instead of host and path

* QMDL store returns single unuploaded entry at a time

* Reset Cargo.lock

---------

Co-authored-by: Markus Unterwaditzer <markus-github@unterwaditzer.net>
This commit is contained in:
John
2026-04-26 11:56:23 -07:00
committed by GitHub
parent 38b1dd3de2
commit 40070b9339
9 changed files with 688 additions and 3 deletions
+1 -1
View File
@@ -40,7 +40,7 @@ image = { version = "0.25.1", default-features = false, features = ["png", "gif
tempfile = "3.10.2"
async_zip = { version = "0.0.17", features = ["tokio"] }
anyhow = "1.0.98"
reqwest = { version = "0.12.20", default-features = false }
reqwest = { version = "0.12.20", default-features = false, features = ["stream"] }
rustls-rustcrypto = { version = "0.0.2-alpha", optional = true }
rustls-post-quantum = { version = "0.2.4", optional = true }
async-trait = "0.1.88"
+38
View File
@@ -50,6 +50,43 @@ pub struct Config {
pub firewall_restrict_outbound: bool,
/// Vector containing additional wifi client firewall ports to open
pub firewall_allowed_ports: Option<Vec<u16>>,
/// Optional WebDAV upload configuration. When unset, no upload worker runs.
pub webdav: Option<WebdavConfig>,
}
/// Configuration for uploading finished QMDL recordings to a WebDAV server.
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(default)]
#[cfg_attr(feature = "apidocs", derive(utoipa::ToSchema))]
pub struct WebdavConfig {
/// WebDAV server base URL, e.g. "https://example.com/remote.php/files/untitaker/my-subfolder/"
pub url: String,
/// Optional username for HTTP Basic auth
pub username: Option<String>,
/// Optional password for HTTP Basic auth
pub password: Option<String>,
/// Timeout (in seconds) for each upload request
pub upload_timeout_secs: u64,
/// How often (in seconds) the worker scans for entries to upload
pub poll_interval_secs: u64,
/// Minimum age (in seconds) an entry must have before it becomes eligible for upload
pub min_age_secs: i64,
/// Delete the file locally after a successful upload
pub delete_on_upload: bool,
}
impl Default for WebdavConfig {
fn default() -> Self {
WebdavConfig {
url: String::new(),
username: None,
password: None,
upload_timeout_secs: 300,
poll_interval_secs: 3600,
min_age_secs: 86400,
delete_on_upload: false,
}
}
}
impl Default for Config {
@@ -74,6 +111,7 @@ impl Default for Config {
dns_servers: None,
firewall_restrict_outbound: true,
firewall_allowed_ports: None,
webdav: None,
}
}
}
+1
View File
@@ -12,6 +12,7 @@ pub mod pcap;
pub mod qmdl_store;
pub mod server;
pub mod stats;
pub mod webdav;
#[cfg(feature = "apidocs")]
use utoipa::OpenApi;
+12
View File
@@ -12,6 +12,8 @@ mod pcap;
mod qmdl_store;
mod server;
mod stats;
mod webdav;
use std::net::SocketAddr;
use std::sync::Arc;
@@ -27,6 +29,7 @@ use crate::server::{
scan_wifi, serve_static, set_config, set_time_offset, test_notification,
};
use crate::stats::{get_qmdl_manifest, get_system_stats};
use crate::webdav::run_webdav_upload_worker;
use wifi_station::WifiStatus;
use analysis::{
@@ -287,6 +290,15 @@ async fn run_with_config(
);
firewall::apply(&config).await;
if let Some(webdav_config) = config.webdav.clone() {
run_webdav_upload_worker(
&task_tracker,
shutdown_token.clone(),
qmdl_store_lock.clone(),
webdav_config.into(),
);
}
let state = Arc::new(ServerState {
config_path: args.config_path.clone(),
config,
+113 -1
View File
@@ -2,7 +2,7 @@ use std::io::{self, ErrorKind};
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use chrono::{DateTime, Local};
use chrono::{DateTime, Local, TimeDelta};
use log::{info, warn};
use rayhunter::util::RuntimeMetadata;
use serde::{Deserialize, Serialize};
@@ -67,6 +67,9 @@ pub struct ManifestEntry {
pub arch: Option<String>,
#[serde(default)]
pub stop_reason: Option<String>,
/// When the manifest was uploaded to a WebDAV server
#[cfg_attr(feature = "apidocs", schema(value_type = String))]
pub upload_time: Option<DateTime<Local>>,
}
impl ManifestEntry {
@@ -82,6 +85,7 @@ impl ManifestEntry {
system_os: Some(metadata.system_os),
arch: Some(metadata.arch),
stop_reason: None,
upload_time: None,
}
}
@@ -212,6 +216,7 @@ impl RecordingStore {
system_os: None,
arch: None,
stop_reason: None,
upload_time: None,
});
}
@@ -342,6 +347,23 @@ impl RecordingStore {
Ok(())
}
pub fn get_next_unuploaded_entry(&self, min_age: TimeDelta) -> Option<String> {
let now = rayhunter::clock::get_adjusted_now();
self.manifest
.entries
.iter()
.filter_map(|entry| {
if self.is_current_entry(&entry.name) || entry.upload_time.is_some() {
return None;
}
let age = now - entry.last_message_time.unwrap_or(entry.start_time);
(age > min_age).then_some((&entry.name, age))
})
.max_by_key(|(_, age)| *age)
.map(|(name, _)| name.clone())
}
// Finds an entry by filename
pub fn entry_for_name(&self, name: &str) -> Option<(usize, &ManifestEntry)> {
let entry_index = self
@@ -368,6 +390,22 @@ impl RecordingStore {
Ok(())
}
pub async fn mark_entry_as_uploaded(
&mut self,
name: &str,
upload_time: DateTime<Local>,
) -> Result<(), RecordingStoreError> {
let entry_index = self
.manifest
.entries
.iter()
.position(|entry| entry.name == name)
.ok_or(RecordingStoreError::NoSuchEntryError)?;
self.manifest.entries[entry_index].upload_time = Some(upload_time);
self.write_manifest().await?;
Ok(())
}
pub fn is_current_entry(&self, name: &str) -> bool {
match self.current_entry {
Some(idx) => match self.manifest.entries.get(idx) {
@@ -544,4 +582,78 @@ mod tests {
store.delete_all_entries().await.unwrap();
assert!(store.current_entry.is_none());
}
#[tokio::test]
async fn test_mark_entry_as_uploaded_sets_time_and_persists() {
let dir = make_temp_dir();
let mut store = RecordingStore::create(dir.path()).await.unwrap();
let _ = store.new_entry().await.unwrap();
let name = store.manifest.entries[0].name.clone();
store.close_current_entry().await.unwrap();
let upload_time = Local::now();
store
.mark_entry_as_uploaded(&name, upload_time)
.await
.unwrap();
assert_eq!(store.manifest.entries[0].upload_time, Some(upload_time));
let reloaded = RecordingStore::load(dir.path()).await.unwrap();
assert_eq!(reloaded.manifest.entries[0].upload_time, Some(upload_time));
}
#[tokio::test]
async fn test_mark_entry_as_uploaded_missing_entry() {
let dir = make_temp_dir();
let mut store = RecordingStore::create(dir.path()).await.unwrap();
assert!(matches!(
store.mark_entry_as_uploaded("nope", Local::now()).await,
Err(RecordingStoreError::NoSuchEntryError)
));
}
#[tokio::test]
async fn test_get_next_unuploaded_entry() {
let dir = make_temp_dir();
let mut store = RecordingStore::create(dir.path()).await.unwrap();
for _ in 0..3 {
let _ = store.new_entry().await.unwrap();
}
store.manifest.entries[0].name = "entry-0".to_owned();
store.manifest.entries[0].start_time = Local::now() - TimeDelta::seconds(10);
store.manifest.entries[0].last_message_time = None;
store.manifest.entries[1].name = "entry-1".to_owned();
store.manifest.entries[1].start_time = Local::now() - TimeDelta::seconds(10);
store.manifest.entries[1].last_message_time = Some(Local::now() - TimeDelta::seconds(5));
store.manifest.entries[2].name = "entry-2".to_owned();
store.manifest.entries[2].start_time = Local::now() - TimeDelta::seconds(10);
store.manifest.entries[2].last_message_time = Some(Local::now() - TimeDelta::seconds(1));
assert_eq!(
store.get_next_unuploaded_entry(TimeDelta::seconds(3600)),
None,
);
assert_eq!(
store.get_next_unuploaded_entry(TimeDelta::seconds(3)),
Some("entry-0".to_owned())
);
store
.mark_entry_as_uploaded("entry-0", Local::now())
.await
.unwrap();
assert_eq!(
store.get_next_unuploaded_entry(TimeDelta::seconds(3)),
Some("entry-1".to_owned())
);
store
.mark_entry_as_uploaded("entry-1", Local::now())
.await
.unwrap();
assert_eq!(store.get_next_unuploaded_entry(TimeDelta::seconds(3)), None);
}
}
+446
View File
@@ -0,0 +1,446 @@
use std::fmt::Display;
use std::{sync::Arc, time::Duration};
use chrono::TimeDelta;
use log::{info, warn};
use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE};
use reqwest::{Body, Client, Response};
use tokio::fs::File;
use tokio::join;
use tokio::{select, sync::RwLock, time};
use tokio_util::io::ReaderStream;
use tokio_util::{sync::CancellationToken, task::TaskTracker};
use crate::config::WebdavConfig;
use crate::qmdl_store::RecordingStore;
pub struct WebdavUploadWorkerConfig {
poll_interval: Duration,
min_age: TimeDelta,
url: String,
username: Option<String>,
password: Option<String>,
timeout: Duration,
delete_on_upload: bool,
}
impl From<WebdavConfig> for WebdavUploadWorkerConfig {
fn from(value: WebdavConfig) -> Self {
WebdavUploadWorkerConfig {
poll_interval: Duration::from_secs(value.poll_interval_secs),
min_age: TimeDelta::seconds(value.min_age_secs),
url: value.url,
username: value.username,
password: value.password,
timeout: Duration::from_secs(value.upload_timeout_secs),
delete_on_upload: value.delete_on_upload,
}
}
}
enum FileKind {
Analysis,
Qmdl,
}
impl FileKind {
fn as_extension(&self) -> &'static str {
match self {
FileKind::Analysis => ".ndjson",
FileKind::Qmdl => ".qmdl",
}
}
}
impl Display for FileKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FileKind::Analysis => write!(f, "analysis"),
FileKind::Qmdl => write!(f, "QMDL"),
}
}
}
#[derive(Debug, Clone)]
struct WebDavClient {
client: Client,
url: String,
username: Option<String>,
password: Option<String>,
}
impl WebDavClient {
fn new(
mut url: String,
username: Option<String>,
password: Option<String>,
timeout: Duration,
) -> Result<Self, reqwest::Error> {
if !url.ends_with('/') {
url.push('/');
}
Ok(Self {
client: reqwest::Client::builder().timeout(timeout).build()?,
url,
username,
password,
})
}
async fn try_upload_file(&self, file: File, name: &str) -> anyhow::Result<Response> {
let file_size = file.metadata().await?.len();
let stream = ReaderStream::new(file);
let body = Body::wrap_stream(stream);
let target = format!("{}{}", self.url, name);
let client = self
.client
.put(&target)
.header(CONTENT_TYPE, "application/octet-stream")
.header(CONTENT_LENGTH, file_size);
let client = match (&self.username, &self.password) {
(Some(username), Some(password)) => client.basic_auth(username, Some(password)),
(Some(username), None) => client.basic_auth(username, None::<&str>),
(None, None) => client,
(None, Some(_)) => {
warn!(
"Got WebDAV auth setting with no username but with a password, skipping authentication"
);
client
}
};
let resp = client.body(body).send().await?.error_for_status();
Ok(resp?)
}
}
async fn try_upload_entry(
client: WebDavClient,
store: Arc<RwLock<RecordingStore>>,
entry_name: String,
file_kind: FileKind,
shutdown_token: CancellationToken,
) -> Option<()> {
let read_lock = store.read().await;
let entry_idx = read_lock.entry_for_name(&entry_name)?.0;
let file = match file_kind {
FileKind::Analysis => read_lock.open_entry_analysis(entry_idx).await,
FileKind::Qmdl => read_lock.open_entry_qmdl(entry_idx).await,
};
drop(read_lock);
let Ok(file) = file.map_err(|err| {
warn!(
"Unable to open entry: {} {} file: {:?}",
entry_name, file_kind, err
)
}) else {
return None;
};
let file_name = format!("{}{}", entry_name, file_kind.as_extension());
let res = select! {
_ = shutdown_token.cancelled() => {
warn!(
"Cancelling upload for entry {} {} file: received shutdown signal",
entry_name, file_kind
);
return None;
},
res = client.try_upload_file(file, &file_name) => res,
};
match res {
Ok(_) => {
info!("Uploaded {} file for entry {}", file_kind, entry_name);
Some(())
}
Err(err) => {
warn!(
"Failed to upload {} file for entry {}: {:?}",
file_kind, entry_name, err
);
None
}
}
}
pub fn run_webdav_upload_worker(
task_tracker: &TaskTracker,
shutdown_token: CancellationToken,
qmdl_store_lock: Arc<RwLock<RecordingStore>>,
config: WebdavUploadWorkerConfig,
) {
task_tracker.spawn(async move {
let mut interval = time::interval(config.poll_interval);
interval.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
let webdav_client = match WebDavClient::new(
config.url,
config.username,
config.password,
config.timeout,
) {
Ok(client) => client,
Err(err) => {
warn!("Unable to create WebDAV client: {:?}", err);
return;
}
};
loop {
select! {
_ = shutdown_token.cancelled() => break,
_ = interval.tick() => {
loop {
let Some(unuploaded_entry) = qmdl_store_lock
.read()
.await
.get_next_unuploaded_entry(config.min_age) else {
break;
};
let (Some(()), Some(())) = join!(
try_upload_entry(
webdav_client.clone(),
qmdl_store_lock.clone(),
unuploaded_entry.clone(),
FileKind::Qmdl,
shutdown_token.clone(),
),
try_upload_entry(
webdav_client.clone(),
qmdl_store_lock.clone(),
unuploaded_entry.clone(),
FileKind::Analysis,
shutdown_token.clone()
),
) else {
break;
};
if config.delete_on_upload {
match qmdl_store_lock.write().await.delete_entry(&unuploaded_entry).await {
Ok(_) => info!("Successfully deleted entry: {} after upload to WebDAV", unuploaded_entry),
Err(err) => warn!("Unable to delete entry: {} after upload to WebDAV: {}", unuploaded_entry, err),
}
} else {
match qmdl_store_lock.write().await.mark_entry_as_uploaded(&unuploaded_entry, rayhunter::clock::get_adjusted_now()).await {
Ok(_) => info!("Successfully marked entry: {} as uploaded", unuploaded_entry),
Err(err) => warn!("Unable to mark entry: {} as uploaded: {}", unuploaded_entry, err),
}
}
}
}
}
}
});
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{
Router,
body::Bytes,
extract::{Path as AxumPath, State},
http::{HeaderMap, StatusCode},
routing::put,
};
use tempfile::Builder;
use tokio::io::AsyncWriteExt;
use tokio::net::TcpListener;
use tokio::sync::Mutex;
#[derive(Clone, Debug)]
struct RecordedPut {
path: String,
auth: Option<String>,
body: Vec<u8>,
}
async fn capture_put(
State(state): State<Arc<Mutex<Vec<RecordedPut>>>>,
AxumPath(path): AxumPath<String>,
headers: HeaderMap,
body: Bytes,
) -> StatusCode {
let auth = headers
.get("authorization")
.and_then(|v| v.to_str().ok())
.map(String::from);
state.lock().await.push(RecordedPut {
path,
auth,
body: body.to_vec(),
});
StatusCode::CREATED
}
async fn setup_webdav_server() -> (Arc<Mutex<Vec<RecordedPut>>>, String) {
crate::crypto_provider::install_default();
let state = Arc::new(Mutex::new(Vec::new()));
let app = Router::new()
.route("/{*path}", put(capture_put))
.with_state(state.clone());
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let url = format!("http://{}/dav", addr);
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
tokio::time::sleep(Duration::from_millis(100)).await;
(state, url)
}
async fn cleanup_worker(shutdown: CancellationToken, tracker: TaskTracker) {
shutdown.cancel();
tracker.close();
tracker.wait().await;
}
async fn make_store_with_closed_entry(
dir: &std::path::Path,
) -> (Arc<RwLock<RecordingStore>>, String) {
let mut store = RecordingStore::create(dir).await.unwrap();
let (mut qmdl_file, mut analysis_file) = store.new_entry().await.unwrap();
qmdl_file.write_all(b"fake qmdl payload").await.unwrap();
qmdl_file.flush().await.unwrap();
analysis_file
.write_all(b"fake ndjson payload")
.await
.unwrap();
analysis_file.flush().await.unwrap();
let entry_index = store.current_entry.unwrap();
let name = store.manifest.entries[entry_index].name.clone();
store.update_entry_qmdl_size(entry_index, 17).await.unwrap();
store.close_current_entry().await.unwrap();
(Arc::new(RwLock::new(store)), name)
}
#[tokio::test]
async fn test_webdav_upload_worker_uploads_entry() {
let (captured, url) = setup_webdav_server().await;
let dir = Builder::new().prefix("webdav_test").tempdir().unwrap();
let (store, entry_name) = make_store_with_closed_entry(dir.path()).await;
let shutdown = CancellationToken::new();
let tracker = TaskTracker::new();
let config = WebdavUploadWorkerConfig {
poll_interval: Duration::from_millis(50),
min_age: TimeDelta::seconds(-1),
url,
username: Some("user".to_string()),
password: Some("password".to_string()),
timeout: Duration::from_secs(1),
delete_on_upload: false,
};
run_webdav_upload_worker(&tracker, shutdown.clone(), store.clone(), config);
tokio::time::sleep(Duration::from_millis(500)).await;
cleanup_worker(shutdown, tracker).await;
let recorded = captured.lock().await;
assert_eq!(recorded.len(), 2);
let paths: Vec<&str> = recorded.iter().map(|r| r.path.as_str()).collect();
let qmdl_path = format!("dav/{}.qmdl", entry_name);
let ndjson_path = format!("dav/{}.ndjson", entry_name);
assert!(paths.contains(&qmdl_path.as_str()));
assert!(paths.contains(&ndjson_path.as_str()));
for put in recorded.iter() {
assert_eq!(put.auth.as_deref(), Some("Basic dXNlcjpwYXNzd29yZA=="));
}
let qmdl_body = recorded
.iter()
.find(|r| r.path == qmdl_path)
.unwrap()
.body
.clone();
let ndjson_body = recorded
.iter()
.find(|r| r.path == ndjson_path)
.unwrap()
.body
.clone();
drop(recorded);
assert_eq!(qmdl_body, b"fake qmdl payload");
assert_eq!(ndjson_body, b"fake ndjson payload");
let store_read = store.read().await;
let (_, entry) = store_read.entry_for_name(&entry_name).unwrap();
assert!(entry.upload_time.is_some());
}
#[tokio::test]
async fn test_webdav_upload_worker_deletes_when_configured() {
let (captured, url) = setup_webdav_server().await;
let dir = Builder::new().prefix("webdav_test").tempdir().unwrap();
let (store, entry_name) = make_store_with_closed_entry(dir.path()).await;
let shutdown = CancellationToken::new();
let tracker = TaskTracker::new();
let config = WebdavUploadWorkerConfig {
poll_interval: Duration::from_millis(50),
min_age: TimeDelta::seconds(-1),
url,
username: None,
password: None,
timeout: Duration::from_secs(1),
delete_on_upload: true,
};
run_webdav_upload_worker(&tracker, shutdown.clone(), store.clone(), config);
tokio::time::sleep(Duration::from_millis(500)).await;
cleanup_worker(shutdown, tracker).await;
assert_eq!(captured.lock().await.len(), 2);
let store_read = store.read().await;
assert!(store_read.entry_for_name(&entry_name).is_none());
}
#[tokio::test]
async fn test_webdav_upload_worker_respects_min_age() {
let (captured, url) = setup_webdav_server().await;
let dir = Builder::new().prefix("webdav_test").tempdir().unwrap();
let (store, entry_name) = make_store_with_closed_entry(dir.path()).await;
let shutdown = CancellationToken::new();
let tracker = TaskTracker::new();
let config = WebdavUploadWorkerConfig {
poll_interval: Duration::from_millis(50),
min_age: TimeDelta::seconds(3600),
url,
username: None,
password: None,
timeout: Duration::from_secs(1),
delete_on_upload: false,
};
run_webdav_upload_worker(&tracker, shutdown.clone(), store.clone(), config);
tokio::time::sleep(Duration::from_millis(500)).await;
cleanup_worker(shutdown, tracker).await;
assert!(captured.lock().await.is_empty());
let store_read = store.read().await;
let (_, entry) = store_read.entry_for_name(&entry_name).unwrap();
assert!(entry.upload_time.is_none());
}
}