Mirror/Momentum-Firmware
1
0
Fork 0
mirror of https://github.com/Next-Flip/Momentum-Firmware.git synced 2026-04-24 03:29:57 -07:00
Code Issues Packages Projects Releases Wiki Activity

Updater: resource compression (#3716)

Browse Source 
* toolbox: compress: moved decompressor implementation to separate func
* toolbox: compress: callback-based api; cli: storage unpack command
* toolbox: compress: separate r/w contexts for stream api
* targets: f18: sync API
* compress: naming fixes & cleanup
* toolbox: compress: using hs buffer size for stream buffers
* toolbox: tar: heatshrink stream mode
* toolbox: compress: docs & small cleanup
* toolbox: tar: header support for .hs; updater: now uses .hs for resources; .hs.tar: now rewindable
* toolbox: compress: fixed hs stream tail handling
* updater: reworked progress for resources cleanup; rebalanced stage weights
* updater: single-pass decompression; scripts: print resources compression ratio
* updater: fixed warnings
* toolbox: tar: doxygen
* docs: update
* docs: info or tarhs format; scripts: added standalone compression/decompression tool for heatshrink-formatted streams
* scripts: tarhs: fixed parameter handling
* cli: storage extract command; toolbox: tar: guess type based on extension
* unit_tests: added test for streamed raw hs decompressor `compress_decode_streamed`
* unit_tests: compress: added extraction test for .tar.hs
* rpc: autodetect compressed archives
* scripts: minor cleanup of common parts
* scripts: update: now using in-memory intermediate tar stream
* scripts: added hs.py wrapper for heatshrink-related ops (single object and directory-as-tar compression)
* scripts: naming fixes
* Toolbox: export compress_config_heatshrink_default as const symbol
* Toolbox: fix various types naming
* Toolbox: more of types naming fixes
* Toolbox: use size_t in compress io callbacks and structures
* UnitTests: update to match new compress API
* Toolbox: proper path_extract_extension usage

Co-authored-by: あく <alleteam@gmail.com>
This commit is contained in:
hedger
2024-06-30 13:38:48 +03:00
committed by GitHub
parent a881816673
commit fcbcb6b5a8
25 changed files with 1339 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
applications/debug/unit_tests/tests/compress/compress_test.c
View File

@@ -1,6 +1,9 @@
#include "../test.h" // IWYU pragma: keep
#include <toolbox/compress.h>
#include <toolbox/md5_calc.h>
#include <toolbox/tar/tar_archive.h>
#include <toolbox/dir_walk.h>
#include <furi.h>
#include <furi_hal.h>
@@ -56,7 +59,7 @@ static void compress_test_reference_comp_decomp() {
furi_record_close(RECORD_STORAGE);
uint8_t* temp_buffer = malloc(1024);
Compress* comp = compress_alloc(1024);
Compress* comp = compress_alloc(CompressTypeHeatshrink, &compress_config_heatshrink_default);
size_t encoded_size = 0;
mu_assert(
@@ -98,7 +101,7 @@ static void compress_test_random_comp_decomp() {
// We only fill half of the buffer with random data, so if anything goes wrong, there's no overflow
static const size_t src_data_size = src_buffer_size / 2;
Compress* comp = compress_alloc(src_buffer_size);
Compress* comp = compress_alloc(CompressTypeHeatshrink, &compress_config_heatshrink_default);
uint8_t* src_buff = malloc(src_buffer_size);
uint8_t* encoded_buff = malloc(encoded_buffer_size);
uint8_t* decoded_buff = malloc(src_buffer_size);
@@ -146,9 +149,200 @@ static void compress_test_random_comp_decomp() {
compress_free(comp);
}
static int32_t hs_unpacker_file_read(void* context, uint8_t* buffer, size_t size) {
File* file = (File*)context;
return storage_file_read(file, buffer, size);
}
static int32_t hs_unpacker_file_write(void* context, uint8_t* buffer, size_t size) {
File* file = (File*)context;
return storage_file_write(file, buffer, size);
}
/*
Source file was generated with:
```python3
import random, string
random.seed(1337)
with open("hsstream.out.bin", "wb") as f:
for c in random.choices(string.printable, k=1024):
for _ in range(random.randint(1, 10)):
f.write(c.encode())
```
It was compressed with heatshrink using the following command:
`python3 -m heatshrink2 compress -w 9 -l 4 hsstream.out.bin hsstream.in.bin`
*/
#define HSSTREAM_IN COMPRESS_UNIT_TESTS_PATH("hsstream.in.bin")
#define HSSTREAM_OUT COMPRESS_UNIT_TESTS_PATH("hsstream.out.bin")
static void compress_test_heatshrink_stream() {
Storage* api = furi_record_open(RECORD_STORAGE);
File* comp_file = storage_file_alloc(api);
File* dest_file = storage_file_alloc(api);
CompressConfigHeatshrink config = {
.window_sz2 = 9,
.lookahead_sz2 = 4,
.input_buffer_sz = 128,
};
Compress* compress = compress_alloc(CompressTypeHeatshrink, &config);
do {
storage_simply_remove(api, HSSTREAM_OUT);
mu_assert(
storage_file_open(comp_file, HSSTREAM_IN, FSAM_READ, FSOM_OPEN_EXISTING),
"Failed to open compressed file");
mu_assert(
storage_file_open(dest_file, HSSTREAM_OUT, FSAM_WRITE, FSOM_OPEN_ALWAYS),
"Failed to open decompressed file");
mu_assert(
compress_decode_streamed(
compress, hs_unpacker_file_read, comp_file, hs_unpacker_file_write, dest_file),
"Decompression failed");
storage_file_close(dest_file);
unsigned char md5[16];
FS_Error file_error;
mu_assert(
md5_calc_file(dest_file, HSSTREAM_OUT, md5, &file_error), "Failed to calculate md5");
const unsigned char expected_md5[16] = {
0xa3,
0x70,
0xe8,
0x8b,
0xa9,
0x42,
0x74,
0xf4,
0xaa,
0x12,
0x8d,
0x41,
0xd2,
0xb6,
0x71,
0xc9};
mu_assert(memcmp(md5, expected_md5, sizeof(md5)) == 0, "MD5 mismatch after decompression");
storage_simply_remove(api, HSSTREAM_OUT);
} while(false);
compress_free(compress);
storage_file_free(comp_file);
storage_file_free(dest_file);
furi_record_close(RECORD_STORAGE);
}
#define HS_TAR_PATH COMPRESS_UNIT_TESTS_PATH("test.ths")
#define HS_TAR_EXTRACT_PATH COMPRESS_UNIT_TESTS_PATH("tar_out")
static bool file_counter(const char* name, bool is_dir, void* context) {
UNUSED(name);
UNUSED(is_dir);
int32_t* n_entries = (int32_t*)context;
(*n_entries)++;
return true;
}
/*
Heatshrink tar file contents and MD5 sums:
file1.txt: 64295676ceed5cce2d0dcac402e4bda4
file2.txt: 188f67f297eedd7bf3d6a4d3c2fc31c4
dir/file3.txt: 34d98ad8135ffe502dba374690136d16
dir/big_file.txt: ee169c1e1791a4d319dbfaefaa850e98
dir/nested_dir/file4.txt: e099fcb2aaa0672375eaedc549247ee6
dir/nested_dir/empty_file.txt: d41d8cd98f00b204e9800998ecf8427e
XOR of all MD5 sums: 92ed5729786d0e1176d047e35f52d376
*/
static void compress_test_heatshrink_tar() {
Storage* api = furi_record_open(RECORD_STORAGE);
TarArchive* archive = tar_archive_alloc(api);
FuriString* path = furi_string_alloc();
FileInfo fileinfo;
File* file = storage_file_alloc(api);
do {
storage_simply_remove_recursive(api, HS_TAR_EXTRACT_PATH);
mu_assert(storage_simply_mkdir(api, HS_TAR_EXTRACT_PATH), "Failed to create extract dir");
mu_assert(
tar_archive_get_mode_for_path(HS_TAR_PATH) == TarOpenModeReadHeatshrink,
"Invalid mode for heatshrink tar");
mu_assert(
tar_archive_open(archive, HS_TAR_PATH, TarOpenModeReadHeatshrink),
"Failed to open heatshrink tar");
int32_t n_entries = 0;
tar_archive_set_file_callback(archive, file_counter, &n_entries);
mu_assert(
tar_archive_unpack_to(archive, HS_TAR_EXTRACT_PATH, NULL),
"Failed to unpack heatshrink tar");
mu_assert(n_entries == 9, "Invalid number of entries in heatshrink tar");
uint8_t md5_total[16] = {0}, md5_file[16];
DirWalk* dir_walk = dir_walk_alloc(api);
mu_assert(dir_walk_open(dir_walk, HS_TAR_EXTRACT_PATH), "Failed to open dirwalk");
while(dir_walk_read(dir_walk, path, &fileinfo) == DirWalkOK) {
if(file_info_is_dir(&fileinfo)) {
continue;
}
mu_assert(
md5_calc_file(file, furi_string_get_cstr(path), md5_file, NULL),
"Failed to calc md5");
for(size_t i = 0; i < 16; i++) {
md5_total[i] ^= md5_file[i];
}
}
dir_walk_free(dir_walk);
static const unsigned char expected_md5[16] = {
0x92,
0xed,
0x57,
0x29,
0x78,
0x6d,
0x0e,
0x11,
0x76,
0xd0,
0x47,
0xe3,
0x5f,
0x52,
0xd3,
0x76};
mu_assert(memcmp(md5_total, expected_md5, sizeof(md5_total)) == 0, "MD5 mismatch");
storage_simply_remove_recursive(api, HS_TAR_EXTRACT_PATH);
} while(false);
storage_file_free(file);
furi_string_free(path);
tar_archive_free(archive);
furi_record_close(RECORD_STORAGE);
}
MU_TEST_SUITE(test_compress) {
MU_RUN_TEST(compress_test_random_comp_decomp);
MU_RUN_TEST(compress_test_reference_comp_decomp);
MU_RUN_TEST(compress_test_heatshrink_stream);
MU_RUN_TEST(compress_test_heatshrink_tar);
}
int run_minunit_test_compress(void) {