Mirror/rayhunter
1
0
Fork 0
mirror of https://github.com/EFForg/rayhunter.git synced 2026-07-01 06:18:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

Add support for compressed QMDL

Browse Source 
Major changes:
* QmdlWriter now outputs gzipped QMDL files by default
* QmdlReader renamed to QmdlMessageReader, and reads both compressed and
  uncompressed QMDL. It no longer requires bounding to avoid reading
  partially written files.
This commit is contained in:
Will Greenberg
2026-03-30 15:56:03 -07:00
committed by Brad Warren
parent f5a0cddc88
commit 94b989c3c0
13 changed files with 641 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
daemon/src/analysis.rs
+24 -22
View File
@@ -1,16 +1,15 @@
use std::sync::Arc;
use std::{cmp, future, pin};
use std::cmp;
use axum::Json;
use axum::{
extract::{Path, State},
http::StatusCode,
};
use futures::TryStreamExt;
use log::{error, info};
use rayhunter::analysis::analyzer::{AnalyzerConfig, EventType, Harness};
use rayhunter::diag::{DataType, MessagesContainer};
use rayhunter::qmdl::QmdlReader;
use rayhunter::diag::{DiagParsingError, Message, MessagesContainer};
use rayhunter::qmdl::QmdlMessageReader;
use serde::Serialize;
use tokio::fs::File;
use tokio::io::{AsyncWriteExt, BufWriter};
@@ -47,7 +46,7 @@ impl AnalysisWriter {
// Runs the analysis harness on the given container, serializing the results
// to the analysis file, returning the whether any warnings were detected
pub async fn analyze(
pub async fn analyze_container(
&mut self,
container: MessagesContainer,
) -> Result<EventType, std::io::Error> {
@@ -62,6 +61,17 @@ impl AnalysisWriter {
Ok(max_type)
}
pub async fn analyze_message(
&mut self,
maybe_qmdl_msg: Result<Message, DiagParsingError>,
) -> Result<EventType, std::io::Error> {
let row = self.harness.analyze_qmdl_message(maybe_qmdl_msg);
if !row.is_empty() {
self.write(&row).await?;
}
Ok(row.get_max_event_type())
}
async fn write<T: Serialize>(&mut self, value: &T) -> Result<(), std::io::Error> {
let mut value_str = serde_json::to_string(value).unwrap();
value_str.push('\n');
@@ -135,7 +145,7 @@ async fn perform_analysis(
analyzer_config: &AnalyzerConfig,
) -> Result<(), String> {
info!("Opening QMDL and analysis file for {name}...");
let (analysis_file, qmdl_file) = {
let (analysis_file, mut qmdl_reader) = {
let mut qmdl_store = qmdl_store_lock.write().await;
let (entry_index, _) = qmdl_store
.entry_for_name(name)
@@ -149,33 +159,25 @@ async fn perform_analysis(
.await
.map_err(|e| format!("{e:?}"))?
.ok_or("QMDL file not found")?;
let qmdl_reader = QmdlMessageReader::new(qmdl_file)
.await
.map_err(|e| format!("{e:?}"))?;
(analysis_file, qmdl_file)
(analysis_file, qmdl_reader)
};
let mut analysis_writer = AnalysisWriter::new(analysis_file, analyzer_config)
.await
.map_err(|e| format!("{e:?}"))?;
let file_size = qmdl_file
.metadata()
.await
.expect("failed to get QMDL file metadata")
.len();
let mut qmdl_reader = QmdlReader::new(qmdl_file, Some(file_size as usize));
let mut qmdl_stream = pin::pin!(
qmdl_reader
.as_stream()
.try_filter(|container| future::ready(container.data_type == DataType::UserSpace))
);
info!("Starting analysis for {name}...");
while let Some(container) = qmdl_stream
.try_next()
while let Some(maybe_message) = qmdl_reader
.get_next_message()
.await
.expect("failed getting QMDL container")
.expect("failed to get message")
{
let _ = analysis_writer
.analyze(container)
.analyze_message(maybe_message)
.await
.map_err(|e| format!("{e:?}"))?;
}
daemon/src/diag.rs
+37 -27
View File
@@ -74,7 +74,7 @@ pub struct DiagTask {
enum DiagState {
Recording {
qmdl_writer: QmdlWriter<File>,
qmdl_writer: Box<QmdlWriter<File>>,
analysis_writer: Box<AnalysisWriter>,
},
Stopped,
@@ -158,7 +158,7 @@ impl DiagTask {
DiskSpaceCheck::Failed => {}
}
let (qmdl_file, analysis_file) = qmdl_store.new_entry(self.gps_mode).await?;
let (qmdl_gz_file, analysis_file) = qmdl_store.new_entry(self.gps_mode).await?;
// For fixed-mode sessions, write the configured coordinates to the storage
// immediately so the per-session GPS is stored durably and isn't affected
@@ -185,13 +185,11 @@ impl DiagTask {
.await
.map_err(RecordingStoreError::WriteFileError)?;
}
self.stop_current_recording().await;
let qmdl_writer = QmdlWriter::new(qmdl_file);
let qmdl_writer = Box::new(QmdlWriter::new(qmdl_gz_file));
let analysis_writer = AnalysisWriter::new(analysis_file, &self.analyzer_config)
.await
.map_err(RecordingStoreError::WriteFileError)?;
self.state = DiagState::Recording {
qmdl_writer,
analysis_writer: Box::new(analysis_writer),
@@ -300,13 +298,23 @@ impl DiagTask {
let mut state = DiagState::Stopped;
std::mem::swap(&mut self.state, &mut state);
if let DiagState::Recording {
analysis_writer, ..
qmdl_writer,
analysis_writer,
..
} = state
{
analysis_writer
.close()
.await
.expect("failed to close analysis writer");
match (qmdl_writer.close().await, analysis_writer.close().await) {
(Ok(()), Ok(())) => {}
(qmdl_result, analysis_result) => {
if let Err(err) = qmdl_result {
error!("failed to close QmdlWriter: {:?}", err);
}
if let Err(err) = analysis_result {
error!("failed to close AnalysisWriter: {:?}", err);
}
panic!();
}
}
}
}
@@ -374,23 +382,25 @@ impl DiagTask {
self.stop(qmdl_store, Some(reason)).await;
return;
}
debug!(
"total QMDL bytes written: {}, updating manifest...",
qmdl_writer.total_written
);
let index = qmdl_store
.current_entry
.expect("DiagDevice had qmdl_writer, but QmdlStore didn't have current entry???");
if let Err(e) = qmdl_store
.update_entry_qmdl_size(index, qmdl_writer.total_written)
.await
{
let reason = format!("failed to update manifest (disk full?): {e}");
error!("{reason}");
self.stop(qmdl_store, Some(reason)).await;
return;
if let Ok(file_size) = qmdl_writer.size().await {
debug!(
"total QMDL bytes written: {}, updating manifest...",
file_size
);
let index = qmdl_store
.current_entry
.expect("DiagDevice had qmdl_writer, but QmdlStore didn't have current entry???");
if let Err(e) = qmdl_store
.update_entry_qmdl_size(index, file_size)
.await
{
let reason = format!("failed to update manifest (disk full?): {e}");
error!("{reason}");
self.stop(qmdl_store, Some(reason)).await;
return;
}
debug!("done!");
}
debug!("done!");
// Extract the latest packet timestamp from this container
if let Some(ts) = container
@@ -407,7 +417,7 @@ impl DiagTask {
let container_bytes: usize = container.messages.iter().map(|m| m.data.len()).sum();
self.bytes_since_space_check += container_bytes;
let max_type = match analysis_writer.analyze(container).await {
let max_type = match analysis_writer.analyze_container(container).await {
Ok(t) => t,
Err(e) => {
warn!("failed to analyze container: {e}");
daemon/src/pcap.rs
+19 -26
View File
@@ -10,12 +10,11 @@ use axum::http::StatusCode;
use axum::http::header::CONTENT_TYPE;
use axum::response::{IntoResponse, Response};
use log::error;
use rayhunter::diag::DataType;
use rayhunter::gsmtap_parser;
use rayhunter::pcap::{GpsPoint, GsmtapPcapWriter};
use rayhunter::qmdl::QmdlReader;
use rayhunter::qmdl::QmdlMessageReader;
use std::sync::Arc;
use tokio::io::{AsyncRead, AsyncWrite, duplex};
use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite, duplex};
use tokio_util::io::ReaderStream;
#[cfg_attr(feature = "apidocs", utoipa::path(
@@ -51,18 +50,20 @@ pub async fn get_pcap(
"QMDL file is empty, try again in a bit!".to_string(),
));
}
let qmdl_size_bytes = entry.qmdl_size_bytes;
let qmdl_file = qmdl_store
.open_file(entry_index, FileKind::Qmdl)
.await
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("{e:?}")))?
.ok_or((StatusCode::NOT_FOUND, "QMDL file not found".to_string()))?;
let qmdl_reader = QmdlMessageReader::new(qmdl_file)
.await
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("{e:?}")))?;
let (reader, writer) = duplex(1024);
let gps_records = load_gps_records_for_entry(&state, entry_index).await;
drop(qmdl_store);
tokio::spawn(async move {
if let Err(e) = generate_pcap_data(writer, qmdl_file, qmdl_size_bytes, gps_records).await {
if let Err(e) = generate_pcap_data(writer, qmdl_reader, gps_records).await {
error!("failed to generate PCAP: {e:?}");
}
});
@@ -131,37 +132,29 @@ fn find_nearest_gps(records: &[GpsRecord], packet_timestamp: i64) -> Option<GpsP
pub async fn generate_pcap_data<R, W>(
writer: W,
qmdl_file: R,
qmdl_size_bytes: usize,
mut reader: QmdlMessageReader<R>,
gps_records: Vec<GpsRecord>,
) -> Result<(), Error>
where
W: AsyncWrite + Unpin + Send,
R: AsyncRead + Unpin,
R: AsyncRead + AsyncSeek + Unpin,
{
let mut pcap_writer = GsmtapPcapWriter::new(writer).await?;
pcap_writer.write_iface_header().await?;
let mut reader = QmdlReader::new(qmdl_file, Some(qmdl_size_bytes));
while let Some(container) = reader.get_next_messages_container().await? {
if container.data_type != DataType::UserSpace {
continue;
}
for maybe_msg in container.messages() {
match maybe_msg {
Ok(msg) => {
let maybe_gsmtap_msg = gsmtap_parser::parse(msg)?;
if let Some((timestamp, gsmtap_msg)) = maybe_gsmtap_msg {
let packet_unix_ts = timestamp.to_datetime().timestamp();
let gps = find_nearest_gps(&gps_records, packet_unix_ts);
pcap_writer
.write_gsmtap_message(gsmtap_msg, timestamp, gps.as_ref())
.await?;
}
while let Some(maybe_msg) = reader.get_next_message().await? {
match maybe_msg {
Ok(msg) => {
let maybe_gsmtap_msg = gsmtap_parser::parse(msg)?;
if let Some((timestamp, gsmtap_msg)) = maybe_gsmtap_msg {
let packet_unix_ts = timestamp.to_datetime().timestamp();
let gps = find_nearest_gps(&gps_records, packet_unix_ts);
pcap_writer
.write_gsmtap_message(gsmtap_msg, timestamp, gps.as_ref())
.await?;
}
Err(e) => error!("error parsing message: {e:?}"),
}
Err(e) => error!("error parsing message: {e:?}"),
}
}
daemon/src/qmdl_store.rs
+25 -14
View File
@@ -55,16 +55,17 @@ impl FileKind {
// List of all possible physical files on disk.
pub const ALL: &'static [FileKind] = &[FileKind::Qmdl, FileKind::Analysis, FileKind::Gps];
pub fn get_filename(&self, entry_name: &str) -> String {
pub fn get_filename(&self, entry_name: &str, qmdl_compressed: bool) -> String {
match self {
FileKind::Qmdl if qmdl_compressed => format!("{}.qmdl.gz", entry_name),
FileKind::Qmdl => format!("{}.qmdl", entry_name),
FileKind::Analysis => format!("{}.ndjson", entry_name),
FileKind::Gps => format!("{}-gps.ndjson", entry_name),
}
}
pub fn get_filepath<P: AsRef<Path>>(&self, entry_name: &str, base_path: P) -> PathBuf {
base_path.as_ref().join(self.get_filename(entry_name))
pub fn get_filepath<P: AsRef<Path>>(&self, entry_name: &str, base_path: P, qmdl_compressed: bool) -> PathBuf {
base_path.as_ref().join(self.get_filename(entry_name, qmdl_compressed))
}
}
@@ -101,7 +102,6 @@ pub struct ManifestEntry {
/// The system time when the last message was recorded to the file
#[cfg_attr(feature = "apidocs", schema(value_type = String))]
pub last_message_time: Option<DateTime<Local>>,
/// The size of the QMDL file in bytes
pub qmdl_size_bytes: usize,
/// The rayhunter daemon version which generated the file
pub rayhunter_version: Option<String>,
@@ -116,6 +116,8 @@ pub struct ManifestEntry {
pub upload_time: Option<DateTime<Local>>,
#[serde(default)]
pub gps_mode: Option<GpsMode>,
#[serde(default)]
pub compressed: bool,
}
impl ManifestEntry {
@@ -133,11 +135,12 @@ impl ManifestEntry {
stop_reason: None,
upload_time: None,
gps_mode: Some(gps_mode),
compressed: true,
}
}
pub fn get_filepath<P: AsRef<Path>>(&self, file_kind: FileKind, path: P) -> PathBuf {
file_kind.get_filepath(&self.name, path)
file_kind.get_filepath(&self.name, path, self.compressed)
}
}
@@ -196,8 +199,9 @@ impl RecordingStore {
}
// Does a best-effort attempt to recover the manifest from a directory of
// QMDL files. We expect these files to be named like "<timestamp>.qmdl",
// and skip any files which don't match that pattern.
// QMDL files. We expect these files to be named like "<timestamp>.qmdl"
// or "<timestamp>.qmdl.gz", and skip any files which don't match that
// pattern.
pub async fn recover<P>(path: P) -> Result<Self, RecordingStoreError>
where
P: AsRef<Path>,
@@ -217,11 +221,14 @@ impl RecordingStore {
continue;
};
if !filename.ends_with(".qmdl") {
let (stem, compressed) = if filename.ends_with(".qmdl") {
(filename.trim_end_matches(".qmdl"), false)
} else if filename.ends_with(".qmdl.gz") {
(filename.trim_end_matches(".qmdl.gz"), true)
} else {
continue;
}
};
let stem = filename.trim_end_matches(".qmdl");
let Ok(start_timestamp) = stem.parse::<i64>() else {
warn!("QMDL file has invalid name {os_filename:?}, skipping");
continue;
@@ -248,6 +255,7 @@ impl RecordingStore {
info!("successfully recovered QMDL entry {os_filename:?}!");
manifest_entries.push(ManifestEntry {
name: stem.to_string(),
compressed,
start_time: start_time.into(),
last_message_time: Some(last_message_time.into()),
qmdl_size_bytes: metadata.size() as usize,
@@ -322,7 +330,7 @@ impl RecordingStore {
file_kind: FileKind,
) -> Result<Option<File>, RecordingStoreError> {
let entry = &self.manifest.entries[entry_index];
let filepath = file_kind.get_filepath(&entry.name, &self.path);
let filepath = file_kind.get_filepath(&entry.name, &self.path, entry.compressed);
match File::open(&filepath).await {
Ok(file) => Ok(Some(file)),
@@ -496,7 +504,7 @@ impl RecordingStore {
self.write_manifest().await?;
for &file_kind in FileKind::ALL {
let filepath = file_kind.get_filepath(&entry_to_delete.name, &self.path);
let filepath = file_kind.get_filepath(&entry_to_delete.name, &self.path, entry_to_delete.compressed);
remove_file_if_exists(&filepath)
.await
.map_err(RecordingStoreError::DeleteFileError)?;
@@ -513,7 +521,7 @@ impl RecordingStore {
'entries: for entry in &self.manifest.entries {
for &file_kind in FileKind::ALL {
let filepath = file_kind.get_filepath(&entry.name, &self.path);
let filepath = file_kind.get_filepath(&entry.name, &self.path, entry.compressed);
if let Err(e) = remove_file_if_exists(&filepath).await {
log::warn!("failed to remove {filepath:?}: {e:?}");
// Some error happened with deleting this entry, abort and go to the next one.
@@ -583,7 +591,10 @@ mod tests {
.entry_for_name(&store.manifest.entries[entry_index].name)
.unwrap();
assert!(entry.last_message_time.is_some());
assert_eq!(store.manifest.entries[entry_index].qmdl_size_bytes, 1000);
assert_eq!(
store.manifest.entries[entry_index].qmdl_size_bytes,
1000
);
assert_eq!(
RecordingStore::read_manifest(dir.path()).await.unwrap(),
store.manifest
daemon/src/server.rs
+113 -36
View File
@@ -11,10 +11,13 @@ use axum::http::{HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use chrono::{DateTime, Local};
use log::{error, warn};
use rayhunter::qmdl::QmdlMessageReader;
use serde::{Deserialize, Serialize};
use tokio::io::AsyncReadExt;
use std::sync::Arc;
use tokio::fs::write;
use tokio::io::{AsyncReadExt, copy, duplex};
use tokio::io::copy;
use tokio::io::duplex;
use tokio::sync::RwLock;
use tokio::sync::mpsc::Sender;
use tokio_util::compat::FuturesAsyncWriteCompatExt;
@@ -81,14 +84,23 @@ pub async fn get_qmdl(
)
})?
.ok_or((StatusCode::NOT_FOUND, "QMDL file not found".to_string()))?;
let limited_qmdl_file = qmdl_file.take(entry.qmdl_size_bytes as u64);
let qmdl_stream = ReaderStream::new(limited_qmdl_file);
let qmdl_reader = QmdlMessageReader::new(qmdl_file)
.await
.map_err(|err| {
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("error reading QMDL file: {err}"),
)
})?;
let headers = [
(CONTENT_TYPE, "application/octet-stream"),
(CONTENT_LENGTH, &entry.qmdl_size_bytes.to_string()),
(
CONTENT_LENGTH,
&entry.qmdl_size_bytes.to_string(),
),
];
let body = Body::from_stream(qmdl_stream);
let body = Body::from_stream(qmdl_reader.into_qmdl_stream());
Ok((headers, body).into_response())
}
@@ -334,7 +346,7 @@ pub async fn get_zip(
Path(entry_name): Path<String>,
) -> Result<Response, (StatusCode, String)> {
let qmdl_idx = entry_name.trim_end_matches(".zip").to_owned();
let (entry_index, qmdl_size_bytes) = {
let (entry_index, compressed, qmdl_file_size) = {
let qmdl_store = state.qmdl_store_lock.read().await;
let (entry_index, entry) = qmdl_store.entry_for_name(&qmdl_idx).ok_or((
StatusCode::NOT_FOUND,
@@ -348,7 +360,7 @@ pub async fn get_zip(
));
}
(entry_index, entry.qmdl_size_bytes)
(entry_index, entry.compressed, entry.qmdl_size_bytes)
};
let qmdl_store_lock = state.qmdl_store_lock.clone();
@@ -377,23 +389,22 @@ pub async fn get_zip(
continue;
};
let entry = ZipEntryBuilder::new(
file_kind.get_filename(&qmdl_idx).into(),
let zip_entry = ZipEntryBuilder::new(
file_kind.get_filename(&qmdl_idx, compressed).into(),
Compression::Stored,
);
// FuturesAsyncWriteCompatExt::compat_write because async-zip's entrystream does
// not impl tokio's AsyncWrite, but only future's AsyncWrite. This can be removed
// once https://github.com/Majored/rs-async-zip/pull/160 is released.
let mut entry_writer = zip.write_entry_stream(entry).await?.compat_write();
let mut entry_writer = zip.write_entry_stream(zip_entry).await?.compat_write();
// Truncating to qmdl_size_bytes is an attempt to ignore partial writes by the diag
// thread.
if file_kind == FileKind::Qmdl {
copy(&mut file.take(qmdl_size_bytes as u64), &mut entry_writer).await?;
copy(&mut file.take(qmdl_file_size as u64), &mut entry_writer).await?;
} else {
copy(&mut file, &mut entry_writer).await?;
}
entry_writer.into_inner().close().await?;
}
@@ -409,13 +420,12 @@ pub async fn get_zip(
.open_file(entry_index, FileKind::Qmdl)
.await?
.ok_or_else(|| anyhow::anyhow!("QMDL file not found"))?
.take(qmdl_size_bytes as u64)
};
let qmdl_reader = QmdlMessageReader::new(qmdl_file_for_pcap).await?;
if let Err(e) = generate_pcap_data(
&mut entry_writer,
qmdl_file_for_pcap,
qmdl_size_bytes,
qmdl_reader,
gps_records,
)
.await
@@ -532,10 +542,14 @@ pub async fn debug_set_display_state(
#[cfg(test)]
mod tests {
use std::io::Cursor;
use super::*;
use crate::config::GpsMode;
use async_zip::base::read::mem::ZipFileReader;
use axum::extract::{Path, State};
use futures::AsyncReadExt;
use rayhunter::{diag::{DataType, HdlcEncapsulatedMessage, Message, MessagesContainer}, qmdl::{QmdlMessageReader, QmdlWriter}};
use tempfile::TempDir;
async fn create_test_qmdl_store() -> (TempDir, Arc<RwLock<crate::qmdl_store::RecordingStore>>) {
@@ -549,24 +563,24 @@ mod tests {
async fn create_test_entry_with_data(
store_lock: &Arc<RwLock<crate::qmdl_store::RecordingStore>>,
test_data: &[u8],
test_data: &MessagesContainer,
) -> String {
let entry_name = {
let mut store = store_lock.write().await;
let (mut qmdl_file, _analysis_file) = store.new_entry(GpsMode::Disabled).await.unwrap();
let (mut qmdl_gz_file, _analysis_file) = store.new_entry(GpsMode::Disabled).await.unwrap();
if !test_data.is_empty() {
use tokio::io::AsyncWriteExt;
qmdl_file.write_all(test_data).await.unwrap();
qmdl_file.flush().await.unwrap();
}
let mut writer = QmdlWriter::new(&mut qmdl_gz_file);
writer.write_container(test_data).await.unwrap();
writer.close().await.unwrap();
let qmdl_file_size = qmdl_gz_file.metadata().await.unwrap().len() as usize;
let current_entry = store.current_entry.unwrap();
let entry = &store.manifest.entries[current_entry];
let entry_name = entry.name.clone();
store
.update_entry_qmdl_size(current_entry, test_data.len())
.update_entry_qmdl_size(current_entry, qmdl_file_size)
.await
.unwrap();
entry_name
@@ -604,17 +618,69 @@ mod tests {
})
}
// valid HDLC encapsulated diag message generated from
// rayhunter::diag::test::get_test_message
fn create_test_container() -> MessagesContainer {
MessagesContainer {
data_type: DataType::UserSpace,
num_messages: 1,
messages: vec![
HdlcEncapsulatedMessage {
len: 39,
data: vec![
16,
0,
32,
0,
32,
0,
192,
176,
26,
165,
245,
135,
118,
35,
2,
1,
20,
14,
48,
0,
160,
0,
2,
8,
0,
0,
217,
15,
5,
0,
0,
0,
0,
1,
0,
10,
13,
196,
126,
],
},
],
}
}
#[tokio::test]
async fn test_get_zip_success() {
let (_temp_dir, store_lock) = create_test_qmdl_store().await;
let test_qmdl_data = vec![0x7E, 0x00, 0x00, 0x00, 0x10, 0x00, 0x7E];
let test_qmdl_data = create_test_container();
let entry_name = create_test_entry_with_data(&store_lock, &test_qmdl_data).await;
let state = create_test_server_state(store_lock);
let result = get_zip(State(state), Path(entry_name.clone())).await;
assert!(result.is_ok());
let response = result.unwrap();
let response = get_zip(State(state), Path(entry_name.clone())).await.unwrap();
let headers = response.headers();
assert_eq!(headers.get("content-type").unwrap(), "application/zip");
@@ -623,21 +689,32 @@ mod tests {
let body_bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap();
let zip_reader = ZipFileReader::new(body_bytes.to_vec()).await.unwrap();
let filenames = zip_reader
.file()
.entries()
let zip_reader_file = zip_reader.file();
let filenames: Vec<String> = zip_reader_file.entries()
.iter()
.map(|entry| entry.filename().as_str().unwrap().to_owned())
.collect::<Vec<String>>();
.map(|entry| entry.filename().as_str().unwrap().to_string())
.collect();
assert_eq!(
filenames,
vec![
format!("{entry_name}.qmdl"),
format!("{entry_name}.qmdl.gz"),
format!("{entry_name}-gps.ndjson"),
format!("{entry_name}.pcapng"),
]
);
let mut qmdl_body = Vec::with_capacity(128);
zip_reader.reader_without_entry(0)
.await
.unwrap()
.read_to_end(&mut qmdl_body)
.await
.unwrap();
let mut qmdl_reader = QmdlMessageReader::new(Cursor::new(qmdl_body)).await.unwrap();
let expected_message = Message::from_hdlc(&test_qmdl_data.messages[0].data).unwrap();
assert_eq!(
qmdl_reader.get_next_message().await.unwrap(),
Some(Ok(expected_message)),
);
}
}
daemon/src/webdav.rs
+4 -3
View File
@@ -102,7 +102,8 @@ async fn try_upload_entry(
shutdown_token: CancellationToken,
) -> Option<()> {
let read_lock = store.read().await;
let entry_idx = read_lock.entry_for_name(&entry_name)?.0;
let (entry_idx, entry) = read_lock.entry_for_name(&entry_name)?;
let compressed = entry.compressed;
let file = read_lock.open_file(entry_idx, file_kind).await;
drop(read_lock);
@@ -118,7 +119,7 @@ async fn try_upload_entry(
}
};
let file_name = file_kind.get_filename(&entry_name);
let file_name = file_kind.get_filename(&entry_name, compressed);
let res = select! {
_ = shutdown_token.cancelled() => {
@@ -331,7 +332,7 @@ mod tests {
let recorded = captured.lock().await;
assert_eq!(recorded.len(), 3);
let paths: Vec<&str> = recorded.iter().map(|r| r.path.as_str()).collect();
let qmdl_path = format!("dav/{}.qmdl", entry_name);
let qmdl_path = format!("dav/{}.qmdl.gz", entry_name);
let ndjson_path = format!("dav/{}.ndjson", entry_name);
let gps_path = format!("dav/{}-gps.ndjson", entry_name);
assert!(paths.contains(&qmdl_path.as_str()));