From b18ac014850a829f2520aa36cfcc8595408021e6 Mon Sep 17 00:00:00 2001 From: WillyJL <49810075+Willy-JL@users.noreply.github.com> Date: Wed, 3 Jul 2024 02:26:59 +0200 Subject: [PATCH] Updater: Gzip resources dejavu, refactor for CompressStreamDecoder API (#152) * Add back uzlib * FBT: Support different resources compress methods * Tar: Detect gzip compress type * Tar: Generic compressed stream refactor * Compress: Shared config params * Fix comment * Compress: Scaffolding for gzip implementation * Compress: Handle errors from allocation * Compress: Gzip support implementation * Tar/Compress: Don't rewind if already at stream cursor * Update changelog --- .gitmodules | 3 + ChangeLog.md | 4 +- .../unit_tests/tests/compress/compress_test.c | 2 +- .../updater/util/update_task_worker_backup.c | 4 +- lib/SConscript | 1 + lib/toolbox/compress.c | 157 +++++++++++++++--- lib/toolbox/compress.h | 19 ++- lib/toolbox/tar/tar_archive.c | 97 +++++++---- lib/toolbox/tar/tar_archive.h | 3 +- lib/uzlib | 1 + lib/uzlib.scons | 28 ++++ scripts/flipper/assets/tarball.py | 32 +++- scripts/update.py | 4 +- targets/f7/target.json | 1 + 14 files changed, 282 insertions(+), 74 deletions(-) create mode 160000 lib/uzlib create mode 100644 lib/uzlib.scons diff --git a/.gitmodules b/.gitmodules index bebd4517c..69d2e627b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -44,3 +44,6 @@ [submodule "documentation/doxygen/doxygen-awesome-css"] path = documentation/doxygen/doxygen-awesome-css url = https://github.com/jothepro/doxygen-awesome-css.git +[submodule "lib/uzlib"] + path = lib/uzlib + url = https://github.com/pfalcon/uzlib.git diff --git a/ChangeLog.md b/ChangeLog.md index 843399810..938580b65 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -21,6 +21,9 @@ - Desktop: Added TV animation from OFW which was missing (internal on OFW) - UL: BadKB: Add Finnish keyboard layout (by @nicou) - OFW: JS: New modules documentation added (by @rnadyrshin) +- Updater: + - OFW: Resource compression refactor, uses heatshrink (by @hedger) + - Adapted gzip to new `CompressStreamDecoder` API, better compression ratio (by @Willy-JL) - Furi: - OFW: Event loop (by @skotopes) - OFW: Thread signals, loader close, loader get app name (by @gsurkov) @@ -60,7 +63,6 @@ - Simpler plugin wrapper +0.5k free flash (by @Willy-JL) - OFW: Minor storage subcommand lookup refactor (by @hedger) - OFW: Furi: Use static synchronisation primitives, prepare for event loop (by @gsurkov & @skotopes) -- OFW: Updater: Resource compression refactored (by @hedger) - OFW: Code Cleanup: Unused includes, useless checks, unused variables, etc... (by @skotopes) ### Fixed: diff --git a/applications/debug/unit_tests/tests/compress/compress_test.c b/applications/debug/unit_tests/tests/compress/compress_test.c index 0f2bd7a03..3e4b4d736 100644 --- a/applications/debug/unit_tests/tests/compress/compress_test.c +++ b/applications/debug/unit_tests/tests/compress/compress_test.c @@ -182,9 +182,9 @@ static void compress_test_heatshrink_stream() { File* dest_file = storage_file_alloc(api); CompressConfigHeatshrink config = { + .base.input_buffer_sz = 128, .window_sz2 = 9, .lookahead_sz2 = 4, - .input_buffer_sz = 128, }; Compress* compress = compress_alloc(CompressTypeHeatshrink, &config); diff --git a/applications/system/updater/util/update_task_worker_backup.c b/applications/system/updater/util/update_task_worker_backup.c index 7c0ff3494..5a69f4654 100644 --- a/applications/system/updater/util/update_task_worker_backup.c +++ b/applications/system/updater/util/update_task_worker_backup.c @@ -156,7 +156,9 @@ static bool update_task_post_update(UpdateTask* update_task) { file_path); CHECK_RESULT(tar_archive_open( - archive, furi_string_get_cstr(file_path), TarOpenModeReadHeatshrink)); + archive, + furi_string_get_cstr(file_path), + tar_archive_get_mode_for_path(furi_string_get_cstr(file_path)))); update_task_cleanup_resources(update_task); diff --git a/lib/SConscript b/lib/SConscript index 84247d53c..656011ccd 100644 --- a/lib/SConscript +++ b/lib/SConscript @@ -39,6 +39,7 @@ libs = env.BuildModules( "lfrfid", "flipper_application", "music_worker", + "uzlib", "mjs", "nanopb", "update_util", diff --git a/lib/toolbox/compress.c b/lib/toolbox/compress.c index 5e794891f..5736023c4 100644 --- a/lib/toolbox/compress.c +++ b/lib/toolbox/compress.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #define TAG "Compress" @@ -16,9 +17,9 @@ #define COMPRESS_ICON_ENCODED_BUFF_SIZE (256u) const CompressConfigHeatshrink compress_config_heatshrink_default = { + .base.input_buffer_sz = COMPRESS_ICON_ENCODED_BUFF_SIZE, .window_sz2 = COMPRESS_EXP_BUFF_SIZE_LOG, .lookahead_sz2 = COMPRESS_LOOKAHEAD_BUFF_SIZE_LOG, - .input_buffer_sz = COMPRESS_ICON_ENCODED_BUFF_SIZE, }; /** Buffer size for input data */ @@ -376,7 +377,7 @@ bool compress_decode( if(!compress->decoder) { CompressConfigHeatshrink* hs_config = (CompressConfigHeatshrink*)compress->config; compress->decoder = heatshrink_decoder_alloc( - hs_config->input_buffer_sz, hs_config->window_sz2, hs_config->lookahead_sz2); + hs_config->base.input_buffer_sz, hs_config->window_sz2, hs_config->lookahead_sz2); } return compress_decode_internal( compress->decoder, data_in, data_in_size, data_out, data_out_size, data_res_size); @@ -391,13 +392,13 @@ bool compress_decode_streamed( CompressConfigHeatshrink* hs_config = (CompressConfigHeatshrink*)compress->config; if(!compress->decoder) { compress->decoder = heatshrink_decoder_alloc( - hs_config->input_buffer_sz, hs_config->window_sz2, hs_config->lookahead_sz2); + hs_config->base.input_buffer_sz, hs_config->window_sz2, hs_config->lookahead_sz2); } heatshrink_decoder_reset(compress->decoder); return compress_decode_stream_internal( compress->decoder, - hs_config->input_buffer_sz, + hs_config->base.input_buffer_sz, read_cb, read_context, write_cb, @@ -406,49 +407,119 @@ bool compress_decode_streamed( //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +typedef struct { + struct uzlib_uncomp uzlib; + CompressStreamDecoder* sd; + uint32_t dict_sz; + uint8_t dict[]; +} gzip_decoder; + +/* At start since uzlib->source_read_cb has no context, so cast uzlib pointer as struct pointer */ +_Static_assert(offsetof(gzip_decoder, uzlib) == 0, "Wrong layout"); +/* At end without specifying size, can be allocated at once with struct */ +_Static_assert(offsetof(gzip_decoder, dict) == sizeof(gzip_decoder), "Wrong layout"); + struct CompressStreamDecoder { - heatshrink_decoder* decoder; size_t stream_position; size_t decode_buffer_size; size_t decode_buffer_position; uint8_t* decode_buffer; CompressIoCallback read_cb; void* read_context; + CompressType type; + union { + heatshrink_decoder* heatshrink; + gzip_decoder* gzip; + } decoder; }; +static int gzip_decoder_read_cb(struct uzlib_uncomp* uzlib) { + gzip_decoder* gz_decoder = (gzip_decoder*)uzlib; + CompressStreamDecoder* sd = gz_decoder->sd; + + int32_t read_size = sd->read_cb(sd->read_context, sd->decode_buffer, sd->decode_buffer_size); + if(read_size <= 0) { + return -1; + } + + uzlib->source = &sd->decode_buffer[1]; /* We will return buffer[0] at exit */ + uzlib->source_limit = sd->decode_buffer + read_size; + return sd->decode_buffer[0]; +} + +static bool gzip_decoder_reset(gzip_decoder* gz_decoder) { + uzlib_uncompress_init(&gz_decoder->uzlib, gz_decoder->dict, gz_decoder->dict_sz); + gz_decoder->uzlib.source = 0; + gz_decoder->uzlib.source_limit = 0; + gz_decoder->uzlib.source_read_cb = gzip_decoder_read_cb; + + int32_t header_res = uzlib_gzip_parse_header(&gz_decoder->uzlib); + if(header_res != TINF_OK) { + return false; + } + + return true; +} + CompressStreamDecoder* compress_stream_decoder_alloc( CompressType type, const void* config, CompressIoCallback read_cb, void* read_context) { - furi_check(type == CompressTypeHeatshrink); + furi_check(type < CompressTypeMAX); furi_check(config); - const CompressConfigHeatshrink* hs_config = (const CompressConfigHeatshrink*)config; + const CompressConfigBase* base_config = config; CompressStreamDecoder* instance = malloc(sizeof(CompressStreamDecoder)); - instance->decoder = heatshrink_decoder_alloc( - hs_config->input_buffer_sz, hs_config->window_sz2, hs_config->lookahead_sz2); + instance->type = type; instance->stream_position = 0; - instance->decode_buffer_size = hs_config->input_buffer_sz; + instance->decode_buffer_size = base_config->input_buffer_sz; instance->decode_buffer_position = 0; - instance->decode_buffer = malloc(hs_config->input_buffer_sz); + instance->decode_buffer = malloc(base_config->input_buffer_sz); instance->read_cb = read_cb; instance->read_context = read_context; + if(type == CompressTypeHeatshrink) { + const CompressConfigHeatshrink* hs_config = config; + heatshrink_decoder* hs_decoder = heatshrink_decoder_alloc( + base_config->input_buffer_sz, hs_config->window_sz2, hs_config->lookahead_sz2); + if(hs_decoder == NULL) { + free(instance->decode_buffer); + free(instance); + return NULL; + } + instance->decoder.heatshrink = hs_decoder; + + } else if(type == CompressTypeGzip) { + const CompressConfigGzip* gz_config = config; + gzip_decoder* gz_decoder = malloc(sizeof(gzip_decoder) + gz_config->dict_sz); + gz_decoder->sd = instance; + gz_decoder->dict_sz = gz_config->dict_sz; + if(!gzip_decoder_reset(gz_decoder)) { + free(gz_decoder); + free(instance->decode_buffer); + free(instance); + return NULL; + } + instance->decoder.gzip = gz_decoder; + } + return instance; } void compress_stream_decoder_free(CompressStreamDecoder* instance) { furi_check(instance); - heatshrink_decoder_free(instance->decoder); + if(instance->type == CompressTypeHeatshrink) { + heatshrink_decoder_free(instance->decoder.heatshrink); + } else if(instance->type == CompressTypeGzip) { + free(instance->decoder.gzip); + } free(instance->decode_buffer); free(instance); } -static bool compress_decode_stream_chunk( +static bool compress_decode_stream_chunk_heatshrink( CompressStreamDecoder* sd, - CompressIoCallback read_cb, - void* read_context, uint8_t* decompressed_chunk, size_t decomp_chunk_size) { HSD_sink_res sink_res; @@ -471,7 +542,7 @@ static bool compress_decode_stream_chunk( do { size_t poll_size = 0; poll_res = heatshrink_decoder_poll( - sd->decoder, decompressed_chunk, decomp_chunk_size, &poll_size); + sd->decoder.heatshrink, decompressed_chunk, decomp_chunk_size, &poll_size); if(poll_res < 0) { return false; } @@ -485,8 +556,8 @@ static bool compress_decode_stream_chunk( } if(can_read_more && (sd->decode_buffer_position < sd->decode_buffer_size)) { - size_t read_size = read_cb( - read_context, + size_t read_size = sd->read_cb( + sd->read_context, &sd->decode_buffer[sd->decode_buffer_position], sd->decode_buffer_size - sd->decode_buffer_position); sd->decode_buffer_position += read_size; @@ -496,7 +567,7 @@ static bool compress_decode_stream_chunk( while(sd->decode_buffer_position && can_sink_more) { size_t sink_size = 0; sink_res = heatshrink_decoder_sink( - sd->decoder, sd->decode_buffer, sd->decode_buffer_position, &sink_size); + sd->decoder.heatshrink, sd->decode_buffer, sd->decode_buffer_position, &sink_size); can_sink_more = sink_res == HSDR_SINK_OK; if(sink_res < 0) { failed = true; @@ -515,6 +586,36 @@ static bool compress_decode_stream_chunk( return decomp_chunk_size == 0; } +static bool compress_decode_stream_chunk_gzip( + CompressStreamDecoder* sd, + uint8_t* decompressed_chunk, + size_t decomp_chunk_size) { + struct uzlib_uncomp* uzlib = &sd->decoder.gzip->uzlib; + uzlib->dest = decompressed_chunk; + uzlib->dest_limit = decompressed_chunk + decomp_chunk_size; + + /* Calls user-provided read_cb via uzlib->source_read_cb configured in gzip_decoder_reset() */ + int32_t res = uzlib_uncompress_chksum(uzlib); + if(res < 0) { + return false; + } + + size_t decomp_size = uzlib->dest - decompressed_chunk; + return decomp_size == decomp_chunk_size; +} + +static bool compress_decode_stream_chunk( + CompressStreamDecoder* sd, + uint8_t* decompressed_chunk, + size_t decomp_chunk_size) { + if(sd->type == CompressTypeHeatshrink) { + return compress_decode_stream_chunk_heatshrink(sd, decompressed_chunk, decomp_chunk_size); + } else if(sd->type == CompressTypeGzip) { + return compress_decode_stream_chunk_gzip(sd, decompressed_chunk, decomp_chunk_size); + } + return false; +} + bool compress_stream_decoder_read( CompressStreamDecoder* instance, uint8_t* data_out, @@ -522,8 +623,7 @@ bool compress_stream_decoder_read( furi_check(instance); furi_check(data_out); - if(compress_decode_stream_chunk( - instance, instance->read_cb, instance->read_context, data_out, data_out_size)) { + if(compress_decode_stream_chunk(instance, data_out, data_out_size)) { instance->stream_position += data_out_size; return true; } @@ -533,9 +633,14 @@ bool compress_stream_decoder_read( bool compress_stream_decoder_seek(CompressStreamDecoder* instance, size_t position) { furi_check(instance); + /* No action required */ + if(position == instance->stream_position) { + return true; + } + /* Check if requested position is ahead of current position we can't rewind the input stream */ - furi_check(position >= instance->stream_position); + furi_check(position > instance->stream_position); /* Read and discard data up to requested position */ uint8_t* dummy_buffer = malloc(instance->decode_buffer_size); @@ -565,7 +670,13 @@ bool compress_stream_decoder_rewind(CompressStreamDecoder* instance) { furi_check(instance); /* Reset decoder and read buffer */ - heatshrink_decoder_reset(instance->decoder); + if(instance->type == CompressTypeHeatshrink) { + heatshrink_decoder_reset(instance->decoder.heatshrink); + } else if(instance->type == CompressTypeGzip) { + if(!gzip_decoder_reset(instance->decoder.gzip)) { + return false; + } + } instance->stream_position = 0; instance->decode_buffer_position = 0; diff --git a/lib/toolbox/compress.h b/lib/toolbox/compress.h index f5862222d..f9cca1fcf 100644 --- a/lib/toolbox/compress.h +++ b/lib/toolbox/compress.h @@ -52,18 +52,35 @@ typedef struct Compress Compress; /** Supported compression types */ typedef enum { CompressTypeHeatshrink = 0, + CompressTypeGzip = 1, + CompressTypeMAX, } CompressType; +/** Base configuration for all compression types */ +typedef struct { + uint16_t input_buffer_sz; +} CompressConfigBase; + /** Configuration for heatshrink compression */ typedef struct { + CompressConfigBase base; uint16_t window_sz2; uint16_t lookahead_sz2; - uint16_t input_buffer_sz; } CompressConfigHeatshrink; +_Static_assert(offsetof(CompressConfigHeatshrink, base) == 0, "Base must be first struct member"); + /** Default configuration for heatshrink compression. Used for image assets. */ extern const CompressConfigHeatshrink compress_config_heatshrink_default; +/** Configuration for gzip compression */ +typedef struct { + CompressConfigBase base; + uint32_t dict_sz; +} CompressConfigGzip; + +_Static_assert(offsetof(CompressConfigGzip, base) == 0, "Base must be first struct member"); + /** Allocate encoder and decoder * * @param type Compression type diff --git a/lib/toolbox/tar/tar_archive.c b/lib/toolbox/tar/tar_archive.c index 951f07a88..43f33ee53 100644 --- a/lib/toolbox/tar/tar_archive.c +++ b/lib/toolbox/tar/tar_archive.c @@ -22,6 +22,8 @@ TarOpenMode tar_archive_get_mode_for_path(const char* path) { if(strcmp(ext, ".ths") == 0) { return TarOpenModeReadHeatshrink; + } else if(strcmp(ext, ".tgz") == 0) { + return TarOpenModeReadGzip; } else { return TarOpenModeRead; } @@ -68,13 +70,18 @@ const struct mtar_ops filesystem_ops = { .close = mtar_storage_file_close, }; -/* Heatshrink stream backend - compressed, read-only */ +/* Compressed stream backend, read-only */ typedef struct { - CompressConfigHeatshrink heatshrink_config; + CompressType type; + union { + CompressConfigBase base; // All other configs start with base + CompressConfigHeatshrink heatshrink; + CompressConfigGzip gzip; + } config; File* stream; CompressStreamDecoder* decoder; -} HeatshrinkStream; +} CompressedStream; /* HSDS 'heatshrink data stream' header magic */ static const uint32_t HEATSHRINK_MAGIC = 0x53445348; @@ -87,42 +94,44 @@ typedef struct { } FURI_PACKED HeatshrinkStreamHeader; _Static_assert(sizeof(HeatshrinkStreamHeader) == 7, "Invalid HeatshrinkStreamHeader size"); -static int mtar_heatshrink_file_close(void* stream) { - HeatshrinkStream* hs_stream = stream; - if(hs_stream) { - if(hs_stream->decoder) { - compress_stream_decoder_free(hs_stream->decoder); +static int mtar_compressed_file_close(void* stream) { + CompressedStream* compressed_stream = stream; + if(compressed_stream) { + if(compressed_stream->decoder) { + compress_stream_decoder_free(compressed_stream->decoder); } - storage_file_close(hs_stream->stream); - storage_file_free(hs_stream->stream); - free(hs_stream); + storage_file_close(compressed_stream->stream); + storage_file_free(compressed_stream->stream); + free(compressed_stream); } return MTAR_ESUCCESS; } -static int mtar_heatshrink_file_read(void* stream, void* data, unsigned size) { - HeatshrinkStream* hs_stream = stream; - bool read_success = compress_stream_decoder_read(hs_stream->decoder, data, size); +static int mtar_compressed_file_read(void* stream, void* data, unsigned size) { + CompressedStream* compressed_stream = stream; + bool read_success = compress_stream_decoder_read(compressed_stream->decoder, data, size); return read_success ? (int)size : MTAR_EREADFAIL; } -static int mtar_heatshrink_file_seek(void* stream, unsigned offset) { - HeatshrinkStream* hs_stream = stream; +static int mtar_compressed_file_seek(void* stream, unsigned offset) { + CompressedStream* compressed_stream = stream; bool success = false; - if(offset == 0) { - success = storage_file_seek(hs_stream->stream, sizeof(HeatshrinkStreamHeader), true) && - compress_stream_decoder_rewind(hs_stream->decoder); + if(offset == 0 && compress_stream_decoder_tell(compressed_stream->decoder) != 0) { + uint32_t rewind_offset = + compressed_stream->type == CompressTypeHeatshrink ? sizeof(HeatshrinkStreamHeader) : 0; + success = storage_file_seek(compressed_stream->stream, rewind_offset, true) && + compress_stream_decoder_rewind(compressed_stream->decoder); } else { - success = compress_stream_decoder_seek(hs_stream->decoder, offset); + success = compress_stream_decoder_seek(compressed_stream->decoder, offset); } return success ? MTAR_ESUCCESS : MTAR_ESEEKFAIL; } -const struct mtar_ops heatshrink_ops = { - .read = mtar_heatshrink_file_read, +const struct mtar_ops compressed_ops = { + .read = mtar_compressed_file_read, .write = NULL, // not supported - .seek = mtar_heatshrink_file_seek, - .close = mtar_heatshrink_file_close, + .seek = mtar_compressed_file_seek, + .close = mtar_compressed_file_close, }; ////////////////////////////////////////////////////////////////////////// @@ -160,6 +169,7 @@ bool tar_archive_open(TarArchive* archive, const char* path, TarOpenMode mode) { open_mode = FSOM_CREATE_ALWAYS; break; case TarOpenModeReadHeatshrink: + case TarOpenModeReadGzip: mtar_access = MTAR_READ; access_mode = FSAM_READ; open_mode = FSOM_OPEN_EXISTING; @@ -174,28 +184,45 @@ bool tar_archive_open(TarArchive* archive, const char* path, TarOpenMode mode) { return false; } - if(compressed) { + if(!compressed) { + mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream); + return true; + } + + CompressedStream* compressed_stream = malloc(sizeof(CompressedStream)); + compressed_stream->stream = stream; + compressed_stream->config.base.input_buffer_sz = FILE_BLOCK_SIZE; + + if(mode == TarOpenModeReadHeatshrink) { /* Read and validate stream header */ HeatshrinkStreamHeader header; if(storage_file_read(stream, &header, sizeof(HeatshrinkStreamHeader)) != sizeof(HeatshrinkStreamHeader) || header.magic != HEATSHRINK_MAGIC) { storage_file_close(stream); + free(compressed_stream); return false; } - HeatshrinkStream* hs_stream = malloc(sizeof(HeatshrinkStream)); - hs_stream->stream = stream; - hs_stream->heatshrink_config.window_sz2 = header.window_sz2; - hs_stream->heatshrink_config.lookahead_sz2 = header.lookahead_sz2; - hs_stream->heatshrink_config.input_buffer_sz = FILE_BLOCK_SIZE; - hs_stream->decoder = compress_stream_decoder_alloc( - CompressTypeHeatshrink, &hs_stream->heatshrink_config, file_read_cb, stream); - mtar_init(&archive->tar, mtar_access, &heatshrink_ops, hs_stream); - } else { - mtar_init(&archive->tar, mtar_access, &filesystem_ops, stream); + compressed_stream->type = CompressTypeHeatshrink; + compressed_stream->config.heatshrink.window_sz2 = header.window_sz2; + compressed_stream->config.heatshrink.lookahead_sz2 = header.lookahead_sz2; + + } else if(mode == TarOpenModeReadGzip) { + compressed_stream->type = CompressTypeGzip; + compressed_stream->config.gzip.dict_sz = 32 * 1024; } + compressed_stream->decoder = compress_stream_decoder_alloc( + compressed_stream->type, &compressed_stream->config, file_read_cb, stream); + if(compressed_stream->decoder == NULL) { + storage_file_close(stream); + free(compressed_stream); + return false; + } + + mtar_init(&archive->tar, mtar_access, &compressed_ops, compressed_stream); + return true; } diff --git a/lib/toolbox/tar/tar_archive.h b/lib/toolbox/tar/tar_archive.h index 3d008e1a4..0858627b7 100644 --- a/lib/toolbox/tar/tar_archive.h +++ b/lib/toolbox/tar/tar_archive.h @@ -17,8 +17,9 @@ typedef struct Storage Storage; typedef enum { TarOpenModeRead = 'r', TarOpenModeWrite = 'w', - /* read-only heatshrink compressed tar */ + /* read-only compressed tar */ TarOpenModeReadHeatshrink = 'h', + TarOpenModeReadGzip = 'g', } TarOpenMode; /** Get expected open mode for archive at the path. diff --git a/lib/uzlib b/lib/uzlib new file mode 160000 index 000000000..6d60d651a --- /dev/null +++ b/lib/uzlib @@ -0,0 +1 @@ +Subproject commit 6d60d651a4499a64f2e5b21b4cc08d98cb84b5c1 diff --git a/lib/uzlib.scons b/lib/uzlib.scons new file mode 100644 index 000000000..d7a3c5340 --- /dev/null +++ b/lib/uzlib.scons @@ -0,0 +1,28 @@ +Import("env") + +env.Append( + CPPPATH=[ + "#/lib/uzlib/src", + ], +) + +libenv = env.Clone(FW_LIB_NAME="uzlib") +libenv.ApplyLibFlags() + +libenv.AppendUnique( + CCFLAGS=[ + "-Wno-redundant-decls", + "-Wno-sign-compare", + ], +) + +sources = [ + File("uzlib/src/adler32.c"), + File("uzlib/src/crc32.c"), + File("uzlib/src/tinfgzip.c"), + File("uzlib/src/tinflate.c"), +] + +lib = libenv.StaticLibrary("${FW_LIB_NAME}", sources) +libenv.Install("${LIB_DIST_DIR}", lib) +Return("lib") diff --git a/scripts/flipper/assets/tarball.py b/scripts/flipper/assets/tarball.py index d9fcbe181..28a5160d3 100644 --- a/scripts/flipper/assets/tarball.py +++ b/scripts/flipper/assets/tarball.py @@ -1,4 +1,5 @@ import io +import gzip import tarfile import heatshrink2 @@ -6,7 +7,9 @@ import heatshrink2 from .heatshrink_stream import HeatshrinkDataStreamHeader FLIPPER_TAR_FORMAT = tarfile.USTAR_FORMAT -TAR_HEATSRINK_EXTENSION = ".ths" + +TAR_HEATSHRINK_EXTENSION = ".ths" +TAR_GZIP_EXTENSION = ".tgz" def tar_sanitizer_filter(tarinfo: tarfile.TarInfo): @@ -17,7 +20,12 @@ def tar_sanitizer_filter(tarinfo: tarfile.TarInfo): def compress_tree_tarball( - src_dir, output_name, filter=tar_sanitizer_filter, hs_window=13, hs_lookahead=6 + src_dir, + output_name, + filter=tar_sanitizer_filter, + hs_window=13, + hs_lookahead=6, + gz_level=9, ): plain_tar = io.BytesIO() with tarfile.open( @@ -27,15 +35,21 @@ def compress_tree_tarball( ) as tarball: tarball.add(src_dir, arcname="", filter=filter) plain_tar.seek(0) - src_data = plain_tar.read() - compressed = heatshrink2.compress( - src_data, window_sz2=hs_window, lookahead_sz2=hs_lookahead - ) - header = HeatshrinkDataStreamHeader(hs_window, hs_lookahead) + if output_name.endswith(TAR_HEATSHRINK_EXTENSION): + compressed = heatshrink2.compress( + src_data, window_sz2=hs_window, lookahead_sz2=hs_lookahead + ) + header = HeatshrinkDataStreamHeader(hs_window, hs_lookahead) + compressed = header.pack() + compressed + + elif output_name.endswith(TAR_GZIP_EXTENSION): + compressed = gzip.compress(src_data, compresslevel=gz_level, mtime=0) + + else: + compressed = src_data + with open(output_name, "wb") as f: - f.write(header.pack()) f.write(compressed) - return len(src_data), len(compressed) diff --git a/scripts/update.py b/scripts/update.py index 09a635e3d..2297cff99 100755 --- a/scripts/update.py +++ b/scripts/update.py @@ -15,6 +15,7 @@ from flipper.assets.coprobin import CoproBinary, get_stack_type from flipper.assets.heatshrink_stream import HeatshrinkDataStreamHeader from flipper.assets.obdata import ObReferenceValues, OptionBytesData from flipper.assets.tarball import compress_tree_tarball, tar_sanitizer_filter +from flipper.assets import tarball from flipper.utils.fff import FlipperFormatFile from slideshow import Main as SlideshowMain @@ -23,9 +24,8 @@ class Main(App): UPDATE_MANIFEST_VERSION = 2 UPDATE_MANIFEST_NAME = "update.fuf" - # No compression, plain tar RESOURCE_TAR_MODE = "w:" - RESOURCE_FILE_NAME = "resources.ths" # .Tar.HeatShrink + RESOURCE_FILE_NAME = "resources" + tarball.TAR_GZIP_EXTENSION RESOURCE_ENTRY_NAME_MAX_LENGTH = 100 WHITELISTED_STACK_TYPES = set( diff --git a/targets/f7/target.json b/targets/f7/target.json index 797956f42..c7bffe45f 100644 --- a/targets/f7/target.json +++ b/targets/f7/target.json @@ -30,6 +30,7 @@ "toolbox", "nfc", "digital_signal", + "uzlib", "pulse_reader", "signal_reader", "microtar",