From 811e4bc4b5e7675dd5eae2b3d023bafc2ea8f0d9 Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Tue, 13 Dec 2022 14:28:57 +0100 Subject: [PATCH] Remove LZMA support For unification, in order not to support two different compression algorithms, the LZMA algorithm was replaced by Brotli. The difference in compression time is negligible: system.tar.br: 26082 system.7z: 25480 Change-Id: Id707be53d7e53a50a6076ce92f70b6812c51de0e --- Dockerfile | 5 +- bsdiff/CMakeLists.txt | 2 +- bsdiff/ss_bsdiff.c | 164 ++----------------------------------- mk_delta/common/bin/CreatePatch.py | 7 +- 4 files changed, 12 insertions(+), 166 deletions(-) diff --git a/Dockerfile b/Dockerfile index e35a19e..48eab8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,6 @@ ENV TZ=UTC ADD bsdiff /bsdiff RUN apt-get update && apt-get -y --no-install-recommends install libbrotli-dev libdivsufsort-dev git cmake build-essential pkg-config -RUN git clone git://git.tizen.org/platform/upstream/lzma-sdk -b tizen /lzma-sdk -RUN cd /lzma-sdk && cmake -DLIB_INSTALL_DIR=/usr/local/lib . && make install RUN cd bsdiff && cmake . && make install FROM ubuntu:20.04 @@ -18,9 +16,8 @@ ADD mk_delta /tota-upg/mk_delta/ ADD scripts /tota-upg/scripts/ ADD recovery /tota-upg/recovery/ COPY --from=build /usr/local/bin/ss_bsdiff /usr/local/bin/ -COPY --from=build /usr/local/lib/liblzma-tool.so.* /usr/local/lib RUN apt-get update && \ - apt-get install -y --no-install-recommends libbrotli1 libdivsufsort3 python-is-python2 python2 python-apt python3-apt python3 python3-pip aria2 p7zip-full attr tar file sudo git && rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends libbrotli1 libdivsufsort3 python-is-python2 python2 python-apt python3-apt python3 python3-pip aria2 brotli attr tar file sudo git && rm -rf /var/lib/apt/lists/* RUN pip3 install requests beautifulsoup4 PyYAML diff --git a/bsdiff/CMakeLists.txt b/bsdiff/CMakeLists.txt index 419d386..fb65ea5 100644 --- a/bsdiff/CMakeLists.txt +++ b/bsdiff/CMakeLists.txt @@ -6,7 +6,7 @@ SET(ss_bsdiff_SRCS ss_bsdiff.c) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/bsdiff) INCLUDE(FindPkgConfig) -pkg_check_modules(${PROJECT_NAME}_pkgs REQUIRED liblzma-tool libdivsufsort libbrotlienc) +pkg_check_modules(${PROJECT_NAME}_pkgs REQUIRED libdivsufsort libbrotlienc) FOREACH(flag ${${PROJECT_NAME}_pkgs_CFLAGS}) SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}") diff --git a/bsdiff/ss_bsdiff.c b/bsdiff/ss_bsdiff.c index d892de2..5bd80bd 100644 --- a/bsdiff/ss_bsdiff.c +++ b/bsdiff/ss_bsdiff.c @@ -43,11 +43,6 @@ #include #include -#include -#include <7zFile.h> -#include <7zVersion.h> -#include -#include #include #define SUFSORT_MOD // Change suffix sorting algorithm from Qsufsort to Divsufsort @@ -115,16 +110,10 @@ struct data_thread { FILE * pfbz2; }; -enum compression_method { - CM_LZMA, - CM_BROTLI, -}; - struct bsdiff_info { const char *old_file; const char *new_file; const char *patch_file; - enum compression_method comp_method; }; struct data_thread data; @@ -507,18 +496,6 @@ const char *kCantWriteMessage = "Can not write output file"; const char *kCantAllocateMessage = "Can not allocate memory"; const char *kDataErrorMessage = "Data error"; -static void *SzAlloc(void *p, size_t size) -{ - p = p; - return MyAlloc(size); -} -static void SzFree(void *p, void *address) -{ - p = p; - MyFree(address); -} -static ISzAlloc g_Alloc = { SzAlloc, SzFree }; - int PrintError(char *buffer, const char *message, int buf_size) { snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), @@ -526,13 +503,6 @@ int PrintError(char *buffer, const char *message, int buf_size) return 1; } -int PrintErrorNumber(char *buffer, SRes val, int buf_size) -{ - snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), - "\nError code: %x\n", (unsigned)val); - return 1; -} - int PrintUserError(char *buffer, int buf_size) { return PrintError(buffer, "Incorrect command", buf_size); @@ -541,91 +511,6 @@ int PrintUserError(char *buffer, int buf_size) #define IN_BUF_SIZE (1 << 16) #define OUT_BUF_SIZE (1 << 16) - -static SRes lzma_encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, char *rs) -{ - CLzmaEncHandle enc; - SRes res; - CLzmaEncProps props; - - rs = rs; - - enc = LzmaEnc_Create(&g_Alloc); - if (enc == 0) - return SZ_ERROR_MEM; - - LzmaEncProps_Init(&props); - res = LzmaEnc_SetProps(enc, &props); - - if (res == SZ_OK) { - Byte header[LZMA_PROPS_SIZE + 8]; - size_t headerSize = LZMA_PROPS_SIZE; - int i; - - res = LzmaEnc_WriteProperties(enc, header, &headerSize); - for (i = 0; i < 8; i++) - header[headerSize++] = (Byte)(fileSize >> (8 * i)); - if (outStream->Write(outStream, header, headerSize) != headerSize) - res = SZ_ERROR_WRITE; - else { - if (res == SZ_OK) - res = LzmaEnc_Encode(enc, outStream, inStream, NULL, &g_Alloc, &g_Alloc); - } - } - LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); - return res; -} - -int lzma_compress(const char *input_file, const char *output_file, char *rs, int rs_size) -{ - assert(input_file); - assert(output_file); - assert(rs); - - CFileSeqInStream inStream; - CFileOutStream outStream; - int res; - - FileSeqInStream_CreateVTable(&inStream); - File_Construct(&inStream.file); - - FileOutStream_CreateVTable(&outStream); - File_Construct(&outStream.file); - - size_t t4 = sizeof(UInt32); - size_t t8 = sizeof(UInt64); - if (t4 != 4 || t8 != 8) - return PrintError(rs, "Incorrect UInt32 or UInt64", rs_size); - - if (InFile_Open(&inStream.file, input_file) != 0) - return PrintError(rs, "Can not open input file", rs_size); - - - if (OutFile_Open(&outStream.file, output_file) != 0) - return PrintError(rs, "Can not open output file", rs_size); - - - UInt64 fileSize; - File_GetLength(&inStream.file, &fileSize); - res = lzma_encode(&outStream.s, &inStream.s, fileSize, rs); - - File_Close(&outStream.file); - File_Close(&inStream.file); - - if (res != SZ_OK) { - if (res == SZ_ERROR_MEM) - return PrintError(rs, kCantAllocateMessage, rs_size); - else if (res == SZ_ERROR_DATA) - return PrintError(rs, kDataErrorMessage, rs_size); - else if (res == SZ_ERROR_WRITE) - return PrintError(rs, kCantWriteMessage, rs_size); - else if (res == SZ_ERROR_READ) - return PrintError(rs, kCantReadMessage, rs_size); - return PrintErrorNumber(rs, res, rs_size); - } - return 0; -} - int brotli_compress_internal(int input_fd, int output_fd, int quality) { int res = -1; @@ -715,7 +600,7 @@ int brotli_compress(const char *input_file, const char *output_file, int quality void print_help(const char *arg0) { assert(arg0); - errx(1, "ss_bsdiff Version 5.0\nUsage: %s [-c ] oldfile newfile patchfile\n", arg0); + errx(1, "ss_bsdiff\nUsage: %s oldfile newfile patchfile\n", arg0); } int parse_args(struct bsdiff_info *info, int argc, char *argv[]) @@ -723,43 +608,20 @@ int parse_args(struct bsdiff_info *info, int argc, char *argv[]) assert(info); assert(argv); - info->comp_method = CM_LZMA; // default compression method - - struct option long_options[] = { - {"compression", optional_argument, NULL, 'c'}, - {0, 0 , 0, 0} - }; - - int opt; - - while ((opt = getopt_long(argc, argv, "c:", long_options, NULL)) != -1) { - switch (opt) { - case 'c': - if (strcmp("lzma", optarg) == 0) - info->comp_method = CM_LZMA; - else if (strcmp("brotli", optarg) == 0) - info->comp_method = CM_BROTLI; - else { - err(1, "Unknown compression method: %s", optarg); - return -1; - } - } - } - - if (optind + 2 >= argc) { + if (3 >= argc) { err(1, "Not enough parameters"); print_help(argv[0]); return -1; } - info->old_file = argv[optind]; - info->new_file = argv[optind+1]; - info->patch_file = argv[optind+2]; + info->old_file = argv[1]; + info->new_file = argv[2]; + info->patch_file = argv[3]; return 0; } -int MY_CDECL main(int argc, char *argv[]) +int main(int argc, char *argv[]) { char rs[800] = { 0 }; @@ -785,19 +647,7 @@ int MY_CDECL main(int argc, char *argv[]) err(1, "bsdiff fails to create delta within timelimit"); } #endif - int res = 0; - switch(info.comp_method) { - case CM_LZMA: - res = lzma_compress(TEMP_PATCH_NAME, info.patch_file, rs, sizeof(rs)); - break; - case CM_BROTLI: - res = brotli_compress(TEMP_PATCH_NAME, info.patch_file, BROTLI_COMPRESSION_QUALITY); - break; - default: - printf("Unknown compression method\n"); - res = -1; - break; - } + int res = brotli_compress(TEMP_PATCH_NAME, info.patch_file, BROTLI_COMPRESSION_QUALITY); if (remove(TEMP_PATCH_NAME) < 0) printf("Failed to remove %s\n", TEMP_PATCH_NAME); diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index 7e19a53..57acceb 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -46,9 +46,9 @@ Catching errors at all stages. SHOULD exit & return error in case of failure COMMON_BIN_PATH = "../../common/bin/" DIFF_UTIL = "/usr/local/bin/ss_bsdiff" -ZIPUTIL = "7z -mf=off a " +ZIPUTIL = "tar -I brotli -cf " NEW_FILES_PATH = "run/upgrade-sysroot" -NEW_FILES_ZIP_NAME = "system.7z" +NEW_FILES_ZIP_NAME = "system.tar.br" SYMLINK_TYPE = "SYM" PART_DOC_EXT = ".txt" ATTR_DOC_EXT = "_attr" + PART_DOC_EXT @@ -64,7 +64,6 @@ EMPTY = "" PARENT_DIR = ".." MEM_REQ = 0 MEM_FILE = "NULL" -COMPRESSION_LZMA = "lzma" COMPRESSION_BROTLI = "brotli" SUPPORT_RENAME = "TRUE" @@ -883,7 +882,7 @@ def main(): # TODO verify if other linux distributions support APT library cache = apt.Cache() - if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed: + if cache['brotli'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed: logging.info('Basic utils installed') else: print("Basic utils missing -- ABORT", file=sys.stderr) -- 2.7.4