From 55c67ee60065212b6d15a68254c1e639f6f3d5e7 Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Tue, 10 May 2022 12:56:16 +0200 Subject: [PATCH 01/16] Use Release field from info.ini for old & new version string Change-Id: I58f34834589787101cd740287954e5271e6e71d4 --- mk_delta/common/bin/mk_part_delta.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk_delta/common/bin/mk_part_delta.sh b/mk_delta/common/bin/mk_part_delta.sh index a345a77..34afabf 100755 --- a/mk_delta/common/bin/mk_part_delta.sh +++ b/mk_delta/common/bin/mk_part_delta.sh @@ -31,13 +31,13 @@ fn_gen_metadata() tar xvf ${OLD_TAR_DIR}/${OLD_TAR_FILE} ${PART_IMG_ORG} sudo mount -o loop ${PART_IMG_ORG} ${TMP_MNT_PNT} - local OLD_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Build=" | sed "s/Build=//" | sed "s/;$//"` + local OLD_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Release=" | sed "s/Release=//" | sed "s/;$//"` sudo umount ${TMP_MNT_PNT} rm -rf ${PART_IMG_ORG} tar xvf ${NEW_TAR_DIR}/${NEW_TAR_FILE} ${PART_IMG_ORG} sudo mount -o loop ${PART_IMG_ORG} ${TMP_MNT_PNT} - local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Build=" | sed "s/Build=//" | sed "s/;$//"` + local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Release=" | sed "s/Release=//" | sed "s/;$//"` local BINOUT="$(LC_ALL=C LANG=C file -b ${TMP_MNT_PNT}/bin/ls)" sudo umount ${TMP_MNT_PNT} rm -rf ${PART_IMG_ORG} -- 2.7.4 From 104b35d10ad5e45716c1885e113cea9e83985b65 Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Wed, 11 May 2022 10:55:47 +0200 Subject: [PATCH 02/16] Use arm64 as architecture string instead of aarch64 It's was requested to be consistent with arm32 naming. Change-Id: I0939e70745bb88354cd47861f5ad0326a6a26971 --- mk_delta/common/bin/mk_part_delta.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk_delta/common/bin/mk_part_delta.sh b/mk_delta/common/bin/mk_part_delta.sh index 34afabf..6fe1d9b 100755 --- a/mk_delta/common/bin/mk_part_delta.sh +++ b/mk_delta/common/bin/mk_part_delta.sh @@ -50,7 +50,7 @@ fn_gen_metadata() *ELF*32-bit*ARM,*) ARCHITECTURE=arm32;; # ELF 64-bit LSB pie executable, ARM aarch64, version 1 (GNU/Linux), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, ... - *ELF*64-bit*ARM*aarch64*) ARCHITECTURE=aarch64;; + *ELF*64-bit*ARM*aarch64*) ARCHITECTURE=arm64;; esac echo "${BUILD_STRING_FILE} : ${OLD_BUILD_STRING}@${NEW_BUILD_STRING}" -- 2.7.4 From ca1a2d529ad1a7031be8cbbd5fc89fcd3700e8f5 Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Fri, 13 May 2022 11:37:48 +0200 Subject: [PATCH 03/16] Save the checksum and image size of PRE_UA images in the update.cfg Change-Id: If6338b6ac94ec8c327f0891ae50cc45ee4be4996 --- mk_delta/common/bin/CreatePatch.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index 6fb8d31..fce79b6 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -131,7 +131,7 @@ def main(): UPDATE_CFG_PATH = '../'+sys.argv[8] GenerateDiffAttr = "TRUE" - elif UPDATE_TYPE in [DELTA_IMG, DELTA_IMG_AB, FULL_IMG]: + elif UPDATE_TYPE in [DELTA_IMG, DELTA_IMG_AB, FULL_IMG, PRE_UA]: if len(sys.argv) == 7: #Use path in better way UPDATE_CFG_PATH = '../'+sys.argv[6] @@ -187,6 +187,7 @@ def main(): print('UPDATE_TYPE ---- EXTRA') elif UPDATE_TYPE == PRE_UA: print('UPDATE_TYPE ---- PRE_UA') + SS_pre_ua_update_checksum(PART_NAME, BASE_NEW, UPDATE_CFG_PATH) else: print('UPDATE_TYPE ---- UNKNOWN FORMAT') @@ -205,6 +206,31 @@ def main(): logging.error('Usage: {} '.format(os.path.basename(sys.argv[0]))) raise +def SS_pre_ua_update_checksum(PART_NAME, BASE_NEW, UPDATE_CFG_PATH): + newsize_d= os.path.getsize(BASE_NEW) + SHA_BIN_DEST= hash_file(BASE_NEW) + + #incase UPDATE CFG is empty + DELTA = PART_NAME + #Should throw error if PART NAME NOT found?? + print("newsize: {} sha_bin_dest: {}".format(newsize_d, SHA_BIN_DEST)) + if not UPDATE_CFG_PATH == EMPTY: + f = open(UPDATE_CFG_PATH, 'r') + lines = f.readlines() + f.close() + f = open(UPDATE_CFG_PATH, 'w') + for line in lines: + ConfigItems = line.split() + if ConfigItems[0] == PART_NAME: + logging.info ('Updating %s config' % PART_NAME) + DELTA = ConfigItems[1] + line = line.rstrip('\n') + line = line.replace(line, line+'\t'+str(0)+'\t\t'+str(newsize_d)+'\t\t'+str(0)+'\t\t'+str(SHA_BIN_DEST)+'\n') + print("new line: {}".format(line)) + f.write(line) + else: + f.write(line) + f.close() def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH): f = open(UPDATE_CFG_PATH, 'r') -- 2.7.4 From a8528a1fdbd7e20bf2874ba00c9bdc6b159877d3 Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Thu, 26 May 2022 16:41:59 +0200 Subject: [PATCH 04/16] Add script to generate full delta To generate the full delta, pass the 'full' argument to the delta-generation.sh: ./scripts/delta-generation.sh . device_XYZ full Change-Id: Ic847f396455b97e07091bf01c62bfb21b5da7ea9 --- mk_delta/common/bin/mk_delta_full.sh | 220 +++++++++++++++++++++++++++++++++++ scripts/delta-generation.sh | 23 +++- 2 files changed, 239 insertions(+), 4 deletions(-) create mode 100755 mk_delta/common/bin/mk_delta_full.sh diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh new file mode 100755 index 0000000..c812b08 --- /dev/null +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +set -u +shopt -s nullglob + +fn_set_default_params() +{ + MY_ID=$(whoami) + DATA_DIR=./data + CFG_DIR=./cfg + DELTA_CFG_PATH=${CFG_DIR}/delta.cfg + UPDATE_CFG_PATH=${DATA_DIR}/update.cfg + UPDATE_INFO_FILE=update-info.ini + UPDATE_INFO_ORIG_PATH=${CFG_DIR}/${UPDATE_INFO_FILE}.in + UPDATE_INFO_PATH=${DATA_DIR}/${UPDATE_INFO_FILE} + SETUP_SCRIPT_PATH=${CFG_DIR}/setup.sh + + LOG_PATH=${DATA_DIR}/Delta.log + BUILD_STRING_FILE=build_string.txt + BUILD_STRING_PATH=${DATA_DIR}/${BUILD_STRING_FILE} + # Getting date and minor version + MONDATE=$(date +%m%d) + i=1 + while [ -d "result/${MONDATE}_${i}" ] + do + i=$(($i + 1)) + done + MONDATE="${MONDATE}_${i}" + echo "MONDATE = ${MONDATE}" + + NEW_TAR_DIR="new_tar" + RESULT_DIR=result/$MONDATE/FW_DELTA + DELTA_DIR=${RESULT_DIR}/DELTA + mkdir -p ${DELTA_DIR} + + DIR_WITH_IMAGES=${DATA_DIR}/images + mkdir -p ${DIR_WITH_IMAGES} +} + +fn_gen_metadata() +{ + ROOTFS_IMG="$1" + + echo "===== Start writing image versions =====" + local TMP_MNT_PNT="TMP_MNT_PNT" + mkdir -p ${TMP_MNT_PNT} + + local OLD_BUILD_STRING="0.0" + + sudo mount -o loop ${ROOTFS_IMG} ${TMP_MNT_PNT} + local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Release=" | sed "s/Release=//" | sed "s/;$//"` + local BINOUT="$(LC_ALL=C LANG=C file -b ${TMP_MNT_PNT}/bin/ls)" + + # Copy upgrade scripts + cp -Rd ${TMP_MNT_PNT}/usr/libexec/upgrade-support/* ${DELTA_DIR} + sudo umount ${TMP_MNT_PNT} + + case "${BINOUT}" in + # ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, ... + *ELF*64-bit*x86-64*) ARCHITECTURE=x86-64;; + + # ELF 32-bit LSB shared object, ARM, EABI5 version, dynamically linked, interpreter /lib/ld-linux.so.3, ... + *ELF*32-bit*ARM,*) ARCHITECTURE=arm32;; + + # ELF 64-bit LSB pie executable, ARM aarch64, version 1 (GNU/Linux), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, ... + *ELF*64-bit*ARM*aarch64*) ARCHITECTURE=arm64;; + esac + + echo "${BUILD_STRING_FILE} : ${OLD_BUILD_STRING}@${NEW_BUILD_STRING}" + echo -n ${OLD_BUILD_STRING}@${NEW_BUILD_STRING} > ${BUILD_STRING_PATH} + + sed -e "s,@SOURCE_VERSION@,${OLD_BUILD_STRING},g" \ + -e "s,@TARGET_VERSION@,${NEW_BUILD_STRING},g" \ + -e "s,@ARCHITECTURE@,${ARCHITECTURE},g" \ + -e "s,@IMAGE_TYPE@,all,g" \ + < "${UPDATE_INFO_ORIG_PATH}" > "${UPDATE_INFO_PATH}" + + echo "update-info.ini:" + cat "${UPDATE_INFO_PATH}" + rm -rf ${TMP_MNT_PNT} + echo "===== Finish writing image versions =====" +} + +fn_read_delta_cfg() +{ + in_file=$1 + + echo "Read ${in_file}" + + if [ ! -r ${in_file} ]; then + echo "file ${in_file} cannot be read" + return 1 + fi + + line_num=0 + while read line + do + # skip comment line + if [ "${line:0:1}" = "#" ]; then + continue + fi + + # skip empty line + line_len=${#line} + if [ "${line_len}" = "0" ]; then + continue + fi + + + fields=(${line}) + PART_NAME[${line_num}]=${fields[0]} + PART_BIN[${line_num}]=${fields[1]} + DELTA_BIN[${line_num}]=${fields[2]} + + line_num=$(($line_num + 1)) + + done < ${in_file} + + PART_NUM=${line_num} + return 0 +} + +fn_extract_all_images() +{ + echo "Extracting archives with images:" + for tar_file in "${DATA_DIR}/${NEW_TAR_DIR}"/*.{tar,tar.gz} + do + echo -e "\t$(basename ${tar_file})" + tar xf "${tar_file}" -C "${DIR_WITH_IMAGES}" + done +} + +fn_copy_image() +{ + PART_NAME="$1" + PART_BIN="$2" + DELTA_BIN="$3" + CUR_DIR=$(pwd) + + echo -ne "Add an image for ${PART_NAME} (${PART_BIN} -> ${DELTA_BIN})" + IMG_SIZE=$(du -b ${DIR_WITH_IMAGES}/${PART_BIN} | awk '{print $1}') + SHA1_SUM=$(sha1sum ${DIR_WITH_IMAGES}/${PART_BIN} | awk '{print $1}') + echo -e "\t\t${IMG_SIZE}\t[${SHA1_SUM}]" + + if [ "${PART_NAME}" == "rootfs" ]; then + fn_gen_metadata ${DIR_WITH_IMAGES}/${PART_BIN} + fi + + sudo mv ${DIR_WITH_IMAGES}/${PART_BIN} ${DELTA_DIR}/${DELTA_BIN} + sudo chown ${MY_ID}:${MY_ID} ${DELTA_DIR}/${PART_BIN} + + echo -e "${PART_NAME}\t${DELTA_BIN}\tPRE_UA\t/dev/null\t0\t0\t${IMG_SIZE}\t0\t${SHA1_SUM}" >> ${UPDATE_CFG_PATH} +} + +fn_make_delta_full() +{ + echo "Make full delta file" + fn_extract_all_images $DIR_WITH_IMAGES + + for ((part_num=0;part_num rpi3 | tw1" + echo " DELTA_TYPE> common | full (default: common)" exit fi TOTA_UPG_PATH=$1 TARGET=$2 -SIGN_PKCS_FILE=$3 -SIGN_PKCS_PASSWORD=$4 + +if [ -z ${3+x} ]; then + DELTA_TYPE="common" +else + DELTA_TYPE=$3 +fi + +if [ "$DELTA_TYPE" == "common" ]; then + SIGN_PKCS_FILE=$4 + SIGN_PKCS_PASSWORD=$5 +fi # Path of downloaded images (old, new) TOTA_UPG_WORK=${TOTA_UPG_PATH}/mk_delta/${TARGET} @@ -57,5 +68,9 @@ cd ${CWD} # Execute mk_delta script CWD=${PWD} cd ${TOTA_UPG_WORK} -../common/bin/mk_delta.sh ${SIGN_PKCS_FILE} ${SIGN_PKCS_PASSWORD} +if [ "$DELTA_TYPE" == "full" ]; then + ../common/bin/mk_delta_full.sh +else + ../common/bin/mk_delta.sh ${SIGN_PKCS_FILE} ${SIGN_PKCS_PASSWORD} +fi cd ${CWD} -- 2.7.4 From 86dfa41dd67e9272e7915dde8a871b2c3197d1ed Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Fri, 27 May 2022 17:15:33 +0200 Subject: [PATCH 05/16] Fix rootfs mount for metadata extraction Change-Id: I63e1203ba98fa1513bb3c3b731cb15f1210d8922 --- mk_delta/common/bin/mk_delta_full.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh index c812b08..57eab97 100755 --- a/mk_delta/common/bin/mk_delta_full.sh +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -47,7 +47,7 @@ fn_gen_metadata() local OLD_BUILD_STRING="0.0" - sudo mount -o loop ${ROOTFS_IMG} ${TMP_MNT_PNT} + sudo mount -o loop,ro ${ROOTFS_IMG} ${TMP_MNT_PNT} local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Release=" | sed "s/Release=//" | sed "s/;$//"` local BINOUT="$(LC_ALL=C LANG=C file -b ${TMP_MNT_PNT}/bin/ls)" -- 2.7.4 From e2d0bce1a0540e86c26bcd12e0cd36a2e74bc5e9 Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Fri, 3 Jun 2022 16:49:43 +0200 Subject: [PATCH 06/16] Add upgrade-apply Change-Id: Idd1a48cf89a17bb74351672094cc1f471a2e973e --- mk_delta/common/bin/mk_delta.sh | 2 +- mk_delta/common/bin/mk_delta_full.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mk_delta/common/bin/mk_delta.sh b/mk_delta/common/bin/mk_delta.sh index c6b1176..a951af8 100755 --- a/mk_delta/common/bin/mk_delta.sh +++ b/mk_delta/common/bin/mk_delta.sh @@ -91,7 +91,6 @@ fn_set_default_params() UPDATE_INFO_FILE=update-info.ini UPDATE_INFO_PATH=./data/${UPDATE_INFO_FILE} SETUP_SCRIPT_PATH=./cfg/setup.sh - DELTA_UA=./data/delta.ua UPDATE_MANAGER=./data/upgrade-trigger.sh LOG_PATH=./data/Delta.log @@ -364,6 +363,7 @@ do #--- extract files which would be appended to delta.tar --- echo "Extract binaries for update from images" fn_extract_from_image rootfs.img /usr/libexec/upgrade-support + fn_extract_from_image rootfs.img /usr/bin/upgrade-apply fn_extract_from_image ramdisk-recovery.img /usr/bin/delta.ua #--- archive result directory --- diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh index 57eab97..4ee572e 100755 --- a/mk_delta/common/bin/mk_delta_full.sh +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -53,6 +53,8 @@ fn_gen_metadata() # Copy upgrade scripts cp -Rd ${TMP_MNT_PNT}/usr/libexec/upgrade-support/* ${DELTA_DIR} + # Copy upgrade-apply + cp ${TMP_MNT_PNT}/usr/bin/upgrade-apply ${DELTA_DIR} sudo umount ${TMP_MNT_PNT} case "${BINOUT}" in -- 2.7.4 From 8db12ad5d9e3caf6f13c75df9cee5adbab6ae434 Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Thu, 9 Jun 2022 16:47:09 +0200 Subject: [PATCH 07/16] Rename upgrade types: Types now include information in the name about what the upgrade is (FULL_IMAGE, DELTA_IMAGE) and when it is to be performed (BEFORE_BOOT_FOTA, AT_BOOT_FOTA): PRE_UA -> FULL_IMAGE:BEFORE_BOOT_FOTA FULL_IMG -> FULL_IMAGE:AT_BOOT_FOTA DELTA_IMG -> DELTA_IMAGE:AT_BOOT_FOTA Change-Id: I31343d64a37156d0ca4673b9150cdfeb97f99ff4 --- mk_delta/artik530_710/cfg/delta.cfg | 4 +-- mk_delta/common/bin/CreatePatch.py | 54 ++++++++---------------------------- mk_delta/common/bin/mk_delta.sh | 4 +-- mk_delta/common/bin/mk_delta_full.sh | 2 +- mk_delta/common/bin/mk_part_delta.sh | 10 ++++--- mk_delta/rpi3/cfg/delta.cfg | 8 +++--- mk_delta/rpi4/cfg/delta.cfg | 8 +++--- mk_delta/tw1/cfg/delta.cfg | 8 +++--- mk_delta/tw2/cfg/delta.cfg | 8 +++--- mk_delta/tw3/cfg/delta.cfg | 8 +++--- 10 files changed, 43 insertions(+), 71 deletions(-) diff --git a/mk_delta/artik530_710/cfg/delta.cfg b/mk_delta/artik530_710/cfg/delta.cfg index d5aaa9b..2eee482 100644 --- a/mk_delta/artik530_710/cfg/delta.cfg +++ b/mk_delta/artik530_710/cfg/delta.cfg @@ -2,5 +2,5 @@ # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p3 0 -ramdisk ramdisk.img ramdisk.img PRE_UA /tmp/boot/ramdisk.img 0 -ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img PRE_UA /tmp/boot/ramdisk-recovery.img 0 +ramdisk ramdisk.img ramdisk.img FULL_IMAGE:BEFORE_BOOT_FOTA /tmp/boot/ramdisk.img 0 +ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /tmp/boot/ramdisk-recovery.img 0 diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index fce79b6..75c371d 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -60,9 +60,8 @@ def global_paths(): global NEW_PREFIX global DIFFPATCH_UTIL global SUPPORT_CONTAINERS - global FULL_IMG - global DELTA_IMG - global DELTA_IMG_AB + global FULL_IMAGE + global DELTA_IMAGE global DELTA_FS global EXTRA global COMMON_BIN_PATH @@ -85,12 +84,10 @@ PART_DOC_EXT = ".txt" DIFF_PREFIX = "diff" DIFF_SUFFIX = ".delta" NEW_PREFIX = 'new' -FULL_IMG = "FULL_IMG" -DELTA_IMG = "DELTA_IMG" -DELTA_IMG_AB = "DELTA_IMG_AB" +FULL_IMAGE = "FULL_IMAGE" +DELTA_IMAGE = "DELTA_IMAGE" DELTA_FS = "DELTA_FS" EXTRA = "EXTRA" -PRE_UA = "PRE_UA" LOGFILE = "Delta.log" VERBATIM_LIST = "Verbatim_List.txt" EMPTY = "" @@ -114,6 +111,7 @@ def main(): if len(sys.argv) < 5: sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER') UPDATE_TYPE = sys.argv[1] + UPDATE_TYPE_S = UPDATE_TYPE.split(":") PART_NAME = sys.argv[2] # lets make this also optional BASE_OLD = sys.argv[3] @@ -123,7 +121,7 @@ def main(): ATTR_NEW = EMPTY UPDATE_CFG_PATH = EMPTY GenerateDiffAttr = "FALSE" - if UPDATE_TYPE == DELTA_FS: + if UPDATE_TYPE_S[0] == DELTA_FS: #instead of arguments check it in outdirectory ? if len(sys.argv) == 9: ATTR_OLD = sys.argv[6] @@ -131,7 +129,7 @@ def main(): UPDATE_CFG_PATH = '../'+sys.argv[8] GenerateDiffAttr = "TRUE" - elif UPDATE_TYPE in [DELTA_IMG, DELTA_IMG_AB, FULL_IMG, PRE_UA]: + elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]: if len(sys.argv) == 7: #Use path in better way UPDATE_CFG_PATH = '../'+sys.argv[6] @@ -165,11 +163,12 @@ def main(): print >> sys.stderr, "Basic utils missing -- ABORT" sys.exit(1) - if UPDATE_TYPE == FULL_IMG: + if UPDATE_TYPE_S[0] == FULL_IMAGE: SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH) - elif UPDATE_TYPE == DELTA_IMG: - SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA) - elif UPDATE_TYPE == DELTA_IMG_AB: + # #### currently does not support LZMA #### + # elif UPDATE_TYPE == DELTA_IMAGE: + # SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA) + elif UPDATE_TYPE_S[0] == DELTA_IMAGE: SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI) elif UPDATE_TYPE == DELTA_FS: AttributeFile = ATTR_NEW @@ -185,9 +184,6 @@ def main(): elif UPDATE_TYPE == EXTRA: print('UPDATE_TYPE ---- EXTRA') - elif UPDATE_TYPE == PRE_UA: - print('UPDATE_TYPE ---- PRE_UA') - SS_pre_ua_update_checksum(PART_NAME, BASE_NEW, UPDATE_CFG_PATH) else: print('UPDATE_TYPE ---- UNKNOWN FORMAT') @@ -206,32 +202,6 @@ def main(): logging.error('Usage: {} '.format(os.path.basename(sys.argv[0]))) raise -def SS_pre_ua_update_checksum(PART_NAME, BASE_NEW, UPDATE_CFG_PATH): - newsize_d= os.path.getsize(BASE_NEW) - SHA_BIN_DEST= hash_file(BASE_NEW) - - #incase UPDATE CFG is empty - DELTA = PART_NAME - #Should throw error if PART NAME NOT found?? - print("newsize: {} sha_bin_dest: {}".format(newsize_d, SHA_BIN_DEST)) - if not UPDATE_CFG_PATH == EMPTY: - f = open(UPDATE_CFG_PATH, 'r') - lines = f.readlines() - f.close() - f = open(UPDATE_CFG_PATH, 'w') - for line in lines: - ConfigItems = line.split() - if ConfigItems[0] == PART_NAME: - logging.info ('Updating %s config' % PART_NAME) - DELTA = ConfigItems[1] - line = line.rstrip('\n') - line = line.replace(line, line+'\t'+str(0)+'\t\t'+str(newsize_d)+'\t\t'+str(0)+'\t\t'+str(SHA_BIN_DEST)+'\n') - print("new line: {}".format(line)) - f.write(line) - else: - f.write(line) - f.close() - def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH): f = open(UPDATE_CFG_PATH, 'r') lines = f.readlines() diff --git a/mk_delta/common/bin/mk_delta.sh b/mk_delta/common/bin/mk_delta.sh index a951af8..a3412c5 100755 --- a/mk_delta/common/bin/mk_delta.sh +++ b/mk_delta/common/bin/mk_delta.sh @@ -159,7 +159,7 @@ fn_process_extra_delta() if [ "${part_name}" = "RAMDISK2" ]; then if [ "${RAMDISK2_FULL_IMG}" = "0" ]; then #--- delta recovery partition --- - ${MK_PART_DELTA} ${part_name} DELTA_IMG ${part_bin} ${delta_bin} + ${MK_PART_DELTA} ${part_name} DELTA_IMAGE:AT_BOOT_FOTA ${part_bin} ${delta_bin} if [ "$?" = "0" ]; then sudo mv data/${PART_OUT} ${RESULT_DIR}/${part_bin} sudo mv ${RESULT_DIR}/${part_bin}/${delta_bin} ${DELTA_DIR} @@ -167,7 +167,7 @@ fn_process_extra_delta() fi if [ "${RAMDISK2_FULL_IMG}" = "1" ]; then #--- full recovery partition --- - ${MK_PART_DELTA} ${part_name} FULL_IMG ${part_bin} ${delta_bin} + ${MK_PART_DELTA} ${part_name} FULL_IMAGE:AT_BOOT_FOTA ${part_bin} ${delta_bin} if [ "$?" = "0" ]; then sudo mv data/${PART_OUT} ${RESULT_DIR}/${part_bin} sudo mv ${RESULT_DIR}/${part_bin}/${part_bin} ${DELTA_DIR} diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh index 4ee572e..857e66e 100755 --- a/mk_delta/common/bin/mk_delta_full.sh +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -151,7 +151,7 @@ fn_copy_image() sudo mv ${DIR_WITH_IMAGES}/${PART_BIN} ${DELTA_DIR}/${DELTA_BIN} sudo chown ${MY_ID}:${MY_ID} ${DELTA_DIR}/${PART_BIN} - echo -e "${PART_NAME}\t${DELTA_BIN}\tPRE_UA\t/dev/null\t0\t0\t${IMG_SIZE}\t0\t${SHA1_SUM}" >> ${UPDATE_CFG_PATH} + echo -e "${PART_NAME}\t${DELTA_BIN}\tFULL_IMAGE:BEFORE_BOOT_FOTA\t/dev/null\t0\t0\t${IMG_SIZE}\t0\t${SHA1_SUM}" >> ${UPDATE_CFG_PATH} } fn_make_delta_full() diff --git a/mk_delta/common/bin/mk_part_delta.sh b/mk_delta/common/bin/mk_part_delta.sh index 6fe1d9b..780b9f4 100755 --- a/mk_delta/common/bin/mk_part_delta.sh +++ b/mk_delta/common/bin/mk_part_delta.sh @@ -576,7 +576,8 @@ NEW_TAR_FILE= fn_get_tar_file_names ${PART_BIN} if [ "z${OLD_TAR_FILE}" = "z" ]; then - if [ ! "${UPDATE_TYPE}" = "FULL_IMG" ] && [ ! "${UPDATE_TYPE}" = "PRE_UA" ]; then + UPDATE_TYPE_S=(${UPDATE_TYPE//:/ }) + if [ ! "${UPDATE_TYPE_S[0]}" = "FULL_IMAGE" ]; then echo "[old] tar file does not exist in ${DATA_DIR}/${OLD_TAR_DIR} which contains ${PART_BIN}" #exit 0, cos this is not an error. exit 0 @@ -606,11 +607,12 @@ if [ ! -d ${OUTPUT_DIR} ]; then mkdir ${OUTPUT_DIR} fi -if [ "${UPDATE_TYPE}" = "FULL_IMG" ] || [ "${UPDATE_TYPE}" = "PRE_UA" ]; then +UPDATE_TYPE_S=(${UPDATE_TYPE//:/ }) +if [ "${UPDATE_TYPE_S[0]}" = "FULL_IMAGE" ]; then fn_mk_full_img -elif [ "${UPDATE_TYPE}" = "DELTA_IMG" ] || [ "${UPDATE_TYPE}" = "DELTA_IMG_AB" ]; then +elif [ "${UPDATE_TYPE_S[0]}" = "DELTA_IMAGE" ]; then fn_mk_delta_img -elif [ "${UPDATE_TYPE}" = "DELTA_FS" ]; then +elif [ "${UPDATE_TYPE_S[0]}" = "DELTA_FS" ]; then fn_mk_delta_fs else echo "Invalid update type" diff --git a/mk_delta/rpi3/cfg/delta.cfg b/mk_delta/rpi3/cfg/delta.cfg index 9b4640c..1228590 100644 --- a/mk_delta/rpi3/cfg/delta.cfg +++ b/mk_delta/rpi3/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset -BOOT boot.img boot.img PRE_UA /dev/mmcblk0p1 0 +BOOT boot.img boot.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p1 0 hal hal.img hal.img/ DELTA_FS /dev/mmcblk0p10 0 -modules modules.img modules.img PRE_UA /dev/mmcblk0p6 0 +modules modules.img modules.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p6 0 rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p2 0 -ramdisk ramdisk.img ramdisk.img FULL_IMG /dev/mmcblk0p7 0 -ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img PRE_UA /dev/mmcblk0p8 0 +ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 +ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p8 0 diff --git a/mk_delta/rpi4/cfg/delta.cfg b/mk_delta/rpi4/cfg/delta.cfg index 9b4640c..1228590 100644 --- a/mk_delta/rpi4/cfg/delta.cfg +++ b/mk_delta/rpi4/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset -BOOT boot.img boot.img PRE_UA /dev/mmcblk0p1 0 +BOOT boot.img boot.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p1 0 hal hal.img hal.img/ DELTA_FS /dev/mmcblk0p10 0 -modules modules.img modules.img PRE_UA /dev/mmcblk0p6 0 +modules modules.img modules.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p6 0 rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p2 0 -ramdisk ramdisk.img ramdisk.img FULL_IMG /dev/mmcblk0p7 0 -ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img PRE_UA /dev/mmcblk0p8 0 +ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 +ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p8 0 diff --git a/mk_delta/tw1/cfg/delta.cfg b/mk_delta/tw1/cfg/delta.cfg index 9706fca..8eb9ac6 100755 --- a/mk_delta/tw1/cfg/delta.cfg +++ b/mk_delta/tw1/cfg/delta.cfg @@ -2,7 +2,7 @@ # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p13 0 -boot zImage delta.boot DELTA_IMG /dev/mmcblk0p6 0 -recovery zImage-recovery zImage-recovery PRE_UA /dev/mmcblk0p7 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMG /dev/mmcblk0p8 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img PRE_UA /dev/mmcblk0p4 0 +boot zImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p6 0 +recovery zImage-recovery zImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 diff --git a/mk_delta/tw2/cfg/delta.cfg b/mk_delta/tw2/cfg/delta.cfg index 7d69d23..a69bb53 100755 --- a/mk_delta/tw2/cfg/delta.cfg +++ b/mk_delta/tw2/cfg/delta.cfg @@ -2,7 +2,7 @@ # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p15 0 -boot dzImage delta.boot DELTA_IMG /dev/mmcblk0p8 0 -recovery dzImage-recovery dzImage-recovery PRE_UA /dev/mmcblk0p9 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMG /dev/mmcblk0p5 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img PRE_UA /dev/mmcblk0p4 0 +boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p9 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p5 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 diff --git a/mk_delta/tw3/cfg/delta.cfg b/mk_delta/tw3/cfg/delta.cfg index 812429c..34ea8dc 100755 --- a/mk_delta/tw3/cfg/delta.cfg +++ b/mk_delta/tw3/cfg/delta.cfg @@ -2,7 +2,7 @@ # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p18 0 -boot dzImage delta.boot DELTA_IMG /dev/mmcblk0p11 0 -recovery dzImage-recovery dzImage-recovery PRE_UA /dev/mmcblk0p12 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMG /dev/mmcblk0p8 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img PRE_UA /dev/mmcblk0p7 0 +boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p11 0 +recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p12 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 -- 2.7.4 From b0e37d6341d8b8bd1477fa5cdebd23c3fde5d889 Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Thu, 9 Jun 2022 14:59:06 +0200 Subject: [PATCH 08/16] Use Date field from info.ini for old & new version string Change-Id: I79861a7d7f08166c191552a1c52b71256c51ecd6 --- mk_delta/common/bin/mk_delta_full.sh | 2 +- mk_delta/common/bin/mk_part_delta.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh index 857e66e..d7f91e1 100755 --- a/mk_delta/common/bin/mk_delta_full.sh +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -48,7 +48,7 @@ fn_gen_metadata() local OLD_BUILD_STRING="0.0" sudo mount -o loop,ro ${ROOTFS_IMG} ${TMP_MNT_PNT} - local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Release=" | sed "s/Release=//" | sed "s/;$//"` + local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Date=" | sed "s/Date=//" | sed "s/;$//"` local BINOUT="$(LC_ALL=C LANG=C file -b ${TMP_MNT_PNT}/bin/ls)" # Copy upgrade scripts diff --git a/mk_delta/common/bin/mk_part_delta.sh b/mk_delta/common/bin/mk_part_delta.sh index 780b9f4..a3ad1bf 100755 --- a/mk_delta/common/bin/mk_part_delta.sh +++ b/mk_delta/common/bin/mk_part_delta.sh @@ -31,13 +31,13 @@ fn_gen_metadata() tar xvf ${OLD_TAR_DIR}/${OLD_TAR_FILE} ${PART_IMG_ORG} sudo mount -o loop ${PART_IMG_ORG} ${TMP_MNT_PNT} - local OLD_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Release=" | sed "s/Release=//" | sed "s/;$//"` + local OLD_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Date=" | sed "s/Date=//" | sed "s/;$//"` sudo umount ${TMP_MNT_PNT} rm -rf ${PART_IMG_ORG} tar xvf ${NEW_TAR_DIR}/${NEW_TAR_FILE} ${PART_IMG_ORG} sudo mount -o loop ${PART_IMG_ORG} ${TMP_MNT_PNT} - local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Release=" | sed "s/Release=//" | sed "s/;$//"` + local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Date=" | sed "s/Date=//" | sed "s/;$//"` local BINOUT="$(LC_ALL=C LANG=C file -b ${TMP_MNT_PNT}/bin/ls)" sudo umount ${TMP_MNT_PNT} rm -rf ${PART_IMG_ORG} -- 2.7.4 From 622dd78cac8d69d721f15180f6448cecdaaecc78 Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Mon, 20 Jun 2022 12:18:12 +0200 Subject: [PATCH 09/16] Unifies the location of the result delta.tar file Change-Id: I2089c6e6f10bb1c462f68efe58faa8bb49d32cef --- mk_delta/common/bin/mk_delta_full.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh index d7f91e1..5512f27 100755 --- a/mk_delta/common/bin/mk_delta_full.sh +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -203,7 +203,7 @@ fn_make_delta_full() #--- archive result directory --- pushd result/$MONDATE/FW_DELTA/DELTA/ echo "tar result directory" - sudo tar cf delta.tar * + sudo tar cf ../delta.tar * popd echo "Making delta binary is End!!" -- 2.7.4 From 007bcdf10bcb8e95eb98f45cd1d94183ba87beda Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Mon, 20 Jun 2022 13:28:31 +0200 Subject: [PATCH 10/16] Save a checksum of all files contained in delta archive Change-Id: I82563afd2ef30784ece716a7b9a090241a159b95 --- mk_delta/common/bin/mk_delta.sh | 1 + mk_delta/common/bin/mk_delta_full.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/mk_delta/common/bin/mk_delta.sh b/mk_delta/common/bin/mk_delta.sh index a3412c5..dc15cce 100755 --- a/mk_delta/common/bin/mk_delta.sh +++ b/mk_delta/common/bin/mk_delta.sh @@ -383,6 +383,7 @@ sudo cp ${COMMON_BINDIR}/unpack.sh ./ # Ensure essential files are at beginning of archive (metadata, scripts and binaries to perform upgrade) touch *.txt *.cfg *.ini *.sh *.ua +sha1sum * > checksum.SHA1 sudo tar --overwrite -cpf ../delta.tar $(ls -1td *) SIGN_PKCS_FILE=$1 diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh index 5512f27..dbdfd79 100755 --- a/mk_delta/common/bin/mk_delta_full.sh +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -202,6 +202,7 @@ fn_make_delta_full() #--- archive result directory --- pushd result/$MONDATE/FW_DELTA/DELTA/ + sha1sum * > checksum.SHA1 echo "tar result directory" sudo tar cf ../delta.tar * popd -- 2.7.4 From a8e1de40958fe37a8dcf62d5d47b9da2c3443bdf Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Thu, 21 Jul 2022 14:48:20 +0200 Subject: [PATCH 11/16] Drop DELTA_FS image generation This commit changes all occurences of DELTA_FS with DELTA_IMAGE. DELTA_FS is considered legacy and should not be used unless necessary. Change-Id: I17dccac5ec532af2db0bb2e385c9871763956b38 --- mk_delta/artik530_710/cfg/delta.cfg | 2 +- mk_delta/rpi3/cfg/delta.cfg | 6 +++--- mk_delta/rpi4/cfg/delta.cfg | 6 +++--- mk_delta/tw1/cfg/delta.cfg | 10 +++++----- mk_delta/tw2/cfg/delta.cfg | 10 +++++----- mk_delta/tw3/cfg/delta.cfg | 10 +++++----- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/mk_delta/artik530_710/cfg/delta.cfg b/mk_delta/artik530_710/cfg/delta.cfg index 2eee482..55dd5b6 100644 --- a/mk_delta/artik530_710/cfg/delta.cfg +++ b/mk_delta/artik530_710/cfg/delta.cfg @@ -1,6 +1,6 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p3 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p3 0 ramdisk ramdisk.img ramdisk.img FULL_IMAGE:BEFORE_BOOT_FOTA /tmp/boot/ramdisk.img 0 ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /tmp/boot/ramdisk-recovery.img 0 diff --git a/mk_delta/rpi3/cfg/delta.cfg b/mk_delta/rpi3/cfg/delta.cfg index 1228590..cab40a7 100644 --- a/mk_delta/rpi3/cfg/delta.cfg +++ b/mk_delta/rpi3/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset BOOT boot.img boot.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p1 0 -hal hal.img hal.img/ DELTA_FS /dev/mmcblk0p10 0 +hal hal.img hal.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p10 0 modules modules.img modules.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p6 0 -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p2 0 -ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p2 0 +ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p8 0 diff --git a/mk_delta/rpi4/cfg/delta.cfg b/mk_delta/rpi4/cfg/delta.cfg index 1228590..09380e1 100644 --- a/mk_delta/rpi4/cfg/delta.cfg +++ b/mk_delta/rpi4/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset BOOT boot.img boot.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p1 0 -hal hal.img hal.img/ DELTA_FS /dev/mmcblk0p10 0 +hal hal.img hal.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p10 0 modules modules.img modules.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p6 0 -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p2 0 -ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p2 0 +ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p8 0 diff --git a/mk_delta/tw1/cfg/delta.cfg b/mk_delta/tw1/cfg/delta.cfg index 8eb9ac6..e8b6ad5 100755 --- a/mk_delta/tw1/cfg/delta.cfg +++ b/mk_delta/tw1/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p13 0 -boot zImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p6 0 -recovery zImage-recovery zImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p13 0 +boot zImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p6 0 +recovery zImage-recovery zImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 diff --git a/mk_delta/tw2/cfg/delta.cfg b/mk_delta/tw2/cfg/delta.cfg index a69bb53..00849c8 100755 --- a/mk_delta/tw2/cfg/delta.cfg +++ b/mk_delta/tw2/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p15 0 -boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 -recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p9 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p5 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p15 0 +boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p9 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p5 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 diff --git a/mk_delta/tw3/cfg/delta.cfg b/mk_delta/tw3/cfg/delta.cfg index 34ea8dc..5521f94 100755 --- a/mk_delta/tw3/cfg/delta.cfg +++ b/mk_delta/tw3/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p18 0 -boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p11 0 -recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p12 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p18 0 +boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p11 0 +recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p12 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 -- 2.7.4 From f4bef151ec4c81db9d3ab79ed4c54cb6fa3d9f0e Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Wed, 27 Jul 2022 12:43:20 +0200 Subject: [PATCH 12/16] Add update-verifier Change-Id: Id074d36297145e07f68d49be8f51feafcc726d5a --- upgrade-verifier/README | 9 +++++ upgrade-verifier/checker.sh | 81 +++++++++++++++++++++++++++++++++++++++++++++ upgrade-verifier/fvt.sh | 28 ++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 upgrade-verifier/README create mode 100755 upgrade-verifier/checker.sh create mode 100755 upgrade-verifier/fvt.sh diff --git a/upgrade-verifier/README b/upgrade-verifier/README new file mode 100644 index 0000000..8dff305 --- /dev/null +++ b/upgrade-verifier/README @@ -0,0 +1,9 @@ +1. Connect the target +2. Flash old image +3. Perform the upgrade using the delta file +4. Wait for the upgrade to complete +2. Run ./fvt.sh + +The fvt.sh script will copy checker.sh and update.cfg from the delta file by sdb and run checker.sh. +checker.sh will verify that the partitions checksum match. + diff --git a/upgrade-verifier/checker.sh b/upgrade-verifier/checker.sh new file mode 100755 index 0000000..4897313 --- /dev/null +++ b/upgrade-verifier/checker.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +CONFIG_PATH=update.cfg +BACKGROUND_COPY_LIST=/hal/etc/upgrade/background_copy.list + + +function is_cloned() { + [[ $(device_board_get_partition_ab_cloned) == "1" ]] +} + +function get_part_labels() { + echo -e "${1}_a\n${1}_b" +} + +function current_part_with_suffix() { + echo "${1}_$(current_suffix)" +} + +function opposite_part_with_suffix() { + echo "${1}_$(opposite_suffix)" +} + +function current_suffix() { + device_board_get_current_partition +} + +function opposite_suffix() { + if [[ $(current_suffix) == "a" ]]; then + echo "b" + else + echo "a" + fi +} + +function verify_part() { + PARTLABEL="$1" + SIZE="$2" + SHA="$3" + device=$(blkid -t PARTLABEL="$PARTLABEL" -o device) + CUR_SHA1=$(head -c "$SIZE" "$device" | sha1sum | awk '{print $1}') + [[ "$CUR_SHA1" == "$SHA" ]] +} + +function verify_update() { + echo -e "\n\nVerify update" + echo "Current slot: $(device_board_get_current_partition)" + while read -r LABEL_NAME DELTA_NAME TYPE DEV OFFSET OLD_SIZE NEW_SIZE OLD_SHA NEW_SHA + do + PARTLABEL=$(current_part_with_suffix "$LABEL_NAME") + echo -n "Checking $LABEL_NAME ($PARTLABEL): " + if verify_part "$PARTLABEL" "$NEW_SIZE" "$NEW_SHA"; then + echo "OK" + else + echo "Incorrect checksum" + fi + done < <(sed -e '/^#/d' -e '/^$/d' "$CONFIG_PATH") +} + +function verify_clone() { + echo -e "\n\nVerify clone" + echo "Opposite slot: $(opposite_suffix)" + while read -r LABEL_NAME + do + read -r SIZE SHA < <(awk "/$LABEL_NAME/ {print \$7 \" \" \$9}" $CONFIG_PATH) + PARTLABEL=$(opposite_part_with_suffix "$LABEL_NAME") + echo -n "Checking $LABEL_NAME ($PARTLABEL): " + if verify_part "$PARTLABEL" "$SIZE" "$SHA"; then + echo "OK" + else + echo "Incorrect checksum" + fi + + done < <(sed -e '/^#/d' -e '/^$/d' "$BACKGROUND_COPY_LIST") +} + +if ! is_cloned; then + clone_partitions.sh +fi + +verify_update +verify_clone diff --git a/upgrade-verifier/fvt.sh b/upgrade-verifier/fvt.sh new file mode 100755 index 0000000..49ef53b --- /dev/null +++ b/upgrade-verifier/fvt.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +DEST_DIR=/opt/usr/verifier +CHECKER_SH=checker.sh + + +if [ "$#" != "1" ] || [ ! -f "$1" ]; then + echo "Use:" + echo " $0 " + exit 1 +fi + +UPLOAD_FILES=( checker.sh update.cfg ) + +echo "Extract update.cfg from $1" +tar xvf "${1}" update.cfg + +echo "Upload files" +{ sleep 0.2; echo tizen; } | sdb shell su -c "mkdir -p $DEST_DIR && chmod 777 $DEST_DIR" +for FILE in "${UPLOAD_FILES[@]}"; do + sdb push "$FILE" "$DEST_DIR" +done + +{ sleep 0.2; echo tizen; } | sdb shell su -c "chmod +x $DEST_DIR/$CHECKER_SH" + +echo "Run verifier" +{ sleep 0.2; echo tizen; } | sdb shell su -c "cd $DEST_DIR && $DEST_DIR/$CHECKER_SH" + -- 2.7.4 From 2f95bbd34951462ad9087e7ea36c29a866f37da5 Mon Sep 17 00:00:00 2001 From: SangYoun Kwak Date: Thu, 18 Aug 2022 10:58:05 +0900 Subject: [PATCH 13/16] Modify upgrade-verifier to verify partitions with image files Change-Id: I7d83092a65ff153e18dd498e40122d6787318094 Signed-off-by: SangYoun Kwak --- upgrade-verifier/README | 7 +-- upgrade-verifier/checker.sh | 81 -------------------------- upgrade-verifier/fvt.sh | 28 --------- upgrade-verifier/upgrade-verifier-on-device.sh | 51 ++++++++++++++++ upgrade-verifier/upgrade-verifier.sh | 75 ++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 113 deletions(-) delete mode 100755 upgrade-verifier/checker.sh delete mode 100755 upgrade-verifier/fvt.sh create mode 100755 upgrade-verifier/upgrade-verifier-on-device.sh create mode 100755 upgrade-verifier/upgrade-verifier.sh diff --git a/upgrade-verifier/README b/upgrade-verifier/README index 8dff305..01b8632 100644 --- a/upgrade-verifier/README +++ b/upgrade-verifier/README @@ -2,8 +2,7 @@ 2. Flash old image 3. Perform the upgrade using the delta file 4. Wait for the upgrade to complete -2. Run ./fvt.sh - -The fvt.sh script will copy checker.sh and update.cfg from the delta file by sdb and run checker.sh. -checker.sh will verify that the partitions checksum match. +2. Run ./upgrade-verifier.sh +The upgrade-verifier.sh script will copy upgrade-verifier-on-device.sh and checksum.txt(created by upgrade-verifier.sh) by sdb and run upgrade-verifier-on-device.sh. +upgrade-verifier-on-device.sh will verify that the partitions checksum match. diff --git a/upgrade-verifier/checker.sh b/upgrade-verifier/checker.sh deleted file mode 100755 index 4897313..0000000 --- a/upgrade-verifier/checker.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -CONFIG_PATH=update.cfg -BACKGROUND_COPY_LIST=/hal/etc/upgrade/background_copy.list - - -function is_cloned() { - [[ $(device_board_get_partition_ab_cloned) == "1" ]] -} - -function get_part_labels() { - echo -e "${1}_a\n${1}_b" -} - -function current_part_with_suffix() { - echo "${1}_$(current_suffix)" -} - -function opposite_part_with_suffix() { - echo "${1}_$(opposite_suffix)" -} - -function current_suffix() { - device_board_get_current_partition -} - -function opposite_suffix() { - if [[ $(current_suffix) == "a" ]]; then - echo "b" - else - echo "a" - fi -} - -function verify_part() { - PARTLABEL="$1" - SIZE="$2" - SHA="$3" - device=$(blkid -t PARTLABEL="$PARTLABEL" -o device) - CUR_SHA1=$(head -c "$SIZE" "$device" | sha1sum | awk '{print $1}') - [[ "$CUR_SHA1" == "$SHA" ]] -} - -function verify_update() { - echo -e "\n\nVerify update" - echo "Current slot: $(device_board_get_current_partition)" - while read -r LABEL_NAME DELTA_NAME TYPE DEV OFFSET OLD_SIZE NEW_SIZE OLD_SHA NEW_SHA - do - PARTLABEL=$(current_part_with_suffix "$LABEL_NAME") - echo -n "Checking $LABEL_NAME ($PARTLABEL): " - if verify_part "$PARTLABEL" "$NEW_SIZE" "$NEW_SHA"; then - echo "OK" - else - echo "Incorrect checksum" - fi - done < <(sed -e '/^#/d' -e '/^$/d' "$CONFIG_PATH") -} - -function verify_clone() { - echo -e "\n\nVerify clone" - echo "Opposite slot: $(opposite_suffix)" - while read -r LABEL_NAME - do - read -r SIZE SHA < <(awk "/$LABEL_NAME/ {print \$7 \" \" \$9}" $CONFIG_PATH) - PARTLABEL=$(opposite_part_with_suffix "$LABEL_NAME") - echo -n "Checking $LABEL_NAME ($PARTLABEL): " - if verify_part "$PARTLABEL" "$SIZE" "$SHA"; then - echo "OK" - else - echo "Incorrect checksum" - fi - - done < <(sed -e '/^#/d' -e '/^$/d' "$BACKGROUND_COPY_LIST") -} - -if ! is_cloned; then - clone_partitions.sh -fi - -verify_update -verify_clone diff --git a/upgrade-verifier/fvt.sh b/upgrade-verifier/fvt.sh deleted file mode 100755 index 49ef53b..0000000 --- a/upgrade-verifier/fvt.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -DEST_DIR=/opt/usr/verifier -CHECKER_SH=checker.sh - - -if [ "$#" != "1" ] || [ ! -f "$1" ]; then - echo "Use:" - echo " $0 " - exit 1 -fi - -UPLOAD_FILES=( checker.sh update.cfg ) - -echo "Extract update.cfg from $1" -tar xvf "${1}" update.cfg - -echo "Upload files" -{ sleep 0.2; echo tizen; } | sdb shell su -c "mkdir -p $DEST_DIR && chmod 777 $DEST_DIR" -for FILE in "${UPLOAD_FILES[@]}"; do - sdb push "$FILE" "$DEST_DIR" -done - -{ sleep 0.2; echo tizen; } | sdb shell su -c "chmod +x $DEST_DIR/$CHECKER_SH" - -echo "Run verifier" -{ sleep 0.2; echo tizen; } | sdb shell su -c "cd $DEST_DIR && $DEST_DIR/$CHECKER_SH" - diff --git a/upgrade-verifier/upgrade-verifier-on-device.sh b/upgrade-verifier/upgrade-verifier-on-device.sh new file mode 100755 index 0000000..9bc2e5b --- /dev/null +++ b/upgrade-verifier/upgrade-verifier-on-device.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +function current_suffix() { + device_board_get_current_partition +} + +function opposite_suffix() { + if [[ $(current_suffix) == "a" ]]; then + echo "b" + else + echo "a" + fi +} + +function verify_partitions() { + local slot=$1 + local checksum_file=$2 + + echo -e "\n\nVerify partition ${slot}" + + while read -r label_name bin_name bin_size bin_sha1; do + local partlabel="${label_name}_${slot}" + local dev_path=$(blkid -t PARTLABEL="${partlabel}" -o device) + + echo -n "Checking ${label_name} (bin_name=${bin_name}, partlabel=${partlabel}, dev_path=${dev_path}): " + + if [[ -z ${dev_path} ]]; then + echo "Invalid partlabel" + continue + fi + + if [[ -z "${bin_name}" ]]; then + echo "No binary found, skip" + continue + fi + + local part_sha1=$(head -c ${bin_size} ${dev_path} | sha1sum | awk '{print $1}') + if [[ "${bin_sha1}" == "${part_sha1}" ]]; then + echo "OK" + else + echo "Incorrect checksum(${bin_sha1} != ${part_sha1})" + fi + done < <(sed -e '/^#/d' -e '/^$/d' "${checksum_file}") +} + +if [ "$(device_board_get_partition_ab_cloned)" != "1" ]; then + clone_partitions.sh +fi + +verify_partitions "$(current_suffix)" "checksums.txt" +verify_partitions "$(opposite_suffix)" "checksums.txt" diff --git a/upgrade-verifier/upgrade-verifier.sh b/upgrade-verifier/upgrade-verifier.sh new file mode 100755 index 0000000..04e7975 --- /dev/null +++ b/upgrade-verifier/upgrade-verifier.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +usage() { + echo "usage: $0 " +} + +create_checksum() { + local bl_img=$1 + local ap_img=$2 + local delta_cfg=$3 + local checksum_file=$4 + local temp_dir=".imgs_tmp" + + if [ -d ${temp_dir} ]; then + echo "${temp_dir} already exist" + return 1 + fi + + echo "Creating ${checksum_file}" + + mkdir ${temp_dir} + tar xfv ${bl_img} -C ${temp_dir} > /dev/null + tar xfv ${ap_img} -C ${temp_dir} > /dev/null + + : > ${checksum_file} + while read -r partition_label bin_name delta_name update_type blk_dev blk_offset; do + local bin_path="${temp_dir}/${bin_name}" + if [ ! -f ${bin_path} ]; then + echo "${bin_name} not exist" + echo "${partition_label}" >> ${checksum_file} + continue + fi + local checksum=$(sha1sum ${bin_path} | awk '{print $1}') + local bin_size=$(stat -c %s "${bin_path}") + echo "${partition_label} ${bin_name} ${bin_size} ${checksum}" >> ${checksum_file} + done < <(sed -e '/^#/d' -e '/^$/d' "${delta_cfg}") + + echo "${checksum_file} created" + rm ${temp_dir} -r + + return 0 +} + +# check parameters +if [ $# -ne 3 ]; then + usage $0 + exit 1 +fi + +BL_IMG=$1 +AP_IMG=$2 +DELTA_CFG=$3 +CHECKSUM_FILE="checksums.txt" +DEST_DIR="/opt/usr/verifier" +CHECKER_SCRIPT="upgrade-verifier-on-device.sh" + +if [ ! -f "${BL_IMG}" ] || [ ! -f "${AP_IMG}" ] || [ ! -f "${DELTA_CFG}" ]; then + usage $0 + exit 1 +fi + +create_checksum ${BL_IMG} ${AP_IMG} ${DELTA_CFG} ${CHECKSUM_FILE} +if [ $? -ne 0 ]; then + echo "failed to create checksum file" + exit 1 +fi + +echo "Upload files" +{ sleep 1; echo tizen; } | sdb shell su -c "mkdir -p $DEST_DIR && chmod 777 $DEST_DIR" +sdb push "${CHECKER_SCRIPT}" "${DEST_DIR}" +sdb push "${CHECKSUM_FILE}" "${DEST_DIR}" +{ sleep 1; echo tizen; } | sdb shell su -c "chmod +x $DEST_DIR/$CHECKER_SCRIPT" + +echo "Run verifier" +{ sleep 1; echo tizen; } | sdb shell su -c "cd $DEST_DIR && $DEST_DIR/$CHECKER_SCRIPT" -- 2.7.4 From c459a175da8efb5d3b379159badee86957d2e0d1 Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Tue, 23 Aug 2022 14:00:14 +0200 Subject: [PATCH 14/16] Update README after repository copy Copied platform/core/system/tota-upg -> upgrade-tools, which will handle delta generation from 7.5 onwards. Change-Id: I0413f75b634cb3907cf787a521ae415e95d36be3 --- README | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README b/README index 0ef4c2f..04819dd 100644 --- a/README +++ b/README @@ -1,7 +1,11 @@ - Tizen System Update - Tizen OTA Update Package Generator (tota-upg) +# Tizen System Update - Tizen System Update is the one of Tizen feature. This implements +This repository was previously known as platorm/core/system/tota-upg, +the "Tizen OTA Upgrade Package Generator". From Tizen 7.5 upgrade is +handled by platform/core/system/upgrade-tools repository. + + +Tizen System Update is the one of Tizen feature. This implements the functionality of firmware update base on OTA mechanism. * Notice of Limitations -- 2.7.4 From f75a5c888ab8b6a25653990d10cbe025bcef2c89 Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Wed, 24 Aug 2022 15:12:17 +0200 Subject: [PATCH 15/16] Import bsdiff from libtota This commit import tools to generate binary deltas from libtota commit 143447ad7 ("ss_bsdiff: Fix to speed up patch generation") Change-Id: I3ef5abd6065b476ffbd682c92bf306edd9a2ea4f --- LICENSE.Apache-2.0 | 202 ++++++++++++ LICENSE.BSD-2-Clause | 23 ++ bsdiff/CMakeLists.txt | 28 ++ bsdiff/ss_brotli_patch.c | 355 ++++++++++++++++++++ bsdiff/ss_brotli_patch.h | 22 ++ bsdiff/ss_bsdiff.c | 805 +++++++++++++++++++++++++++++++++++++++++++++ bsdiff/ss_bspatch.c | 155 +++++++++ bsdiff/ss_bspatch_common.c | 291 ++++++++++++++++ bsdiff/ss_bspatch_common.h | 17 + 9 files changed, 1898 insertions(+) create mode 100644 LICENSE.Apache-2.0 create mode 100644 LICENSE.BSD-2-Clause create mode 100644 bsdiff/CMakeLists.txt create mode 100644 bsdiff/ss_brotli_patch.c create mode 100644 bsdiff/ss_brotli_patch.h create mode 100644 bsdiff/ss_bsdiff.c create mode 100644 bsdiff/ss_bspatch.c create mode 100644 bsdiff/ss_bspatch_common.c create mode 100644 bsdiff/ss_bspatch_common.h diff --git a/LICENSE.Apache-2.0 b/LICENSE.Apache-2.0 new file mode 100644 index 0000000..fef8c29 --- /dev/null +++ b/LICENSE.Apache-2.0 @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/LICENSE.BSD-2-Clause b/LICENSE.BSD-2-Clause new file mode 100644 index 0000000..e9cc3cf --- /dev/null +++ b/LICENSE.BSD-2-Clause @@ -0,0 +1,23 @@ +Copyright 2003-2005 Colin Percival +Copyright 2012 Matthew Endsley +All rights reserved + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bsdiff/CMakeLists.txt b/bsdiff/CMakeLists.txt new file mode 100644 index 0000000..ecd7205 --- /dev/null +++ b/bsdiff/CMakeLists.txt @@ -0,0 +1,28 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +PROJECT(ss_bsdiff C) + +SET(ss_bsdiff_SRCS ss_bsdiff.c) +SET(ss_bspatch_SRCS + ss_bspatch_common.c + ss_bspatch.c +) + +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/bsdiff) + +INCLUDE(FindPkgConfig) +pkg_check_modules(${PROJECT_NAME}_pkgs REQUIRED liblzma-tool libdivsufsort libbrotlienc) + +FOREACH(flag ${${PROJECT_NAME}_pkgs_CFLAGS}) + SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}") +ENDFOREACH(flag) + +SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} -I./include") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS}") + +ADD_EXECUTABLE(ss_bsdiff ${ss_bsdiff_SRCS}) +TARGET_LINK_LIBRARIES(ss_bsdiff ${${PROJECT_NAME}_pkgs_LDFLAGS} "-g" "-pthread") +INSTALL(TARGETS ss_bsdiff DESTINATION bin) + +ADD_EXECUTABLE(ss_bspatch ${ss_bspatch_SRCS}) +TARGET_LINK_LIBRARIES(ss_bspatch ${${PROJECT_NAME}_pkgs_LDFLAGS} "-g" "-pthread") +INSTALL(TARGETS ss_bspatch DESTINATION bin) diff --git a/bsdiff/ss_brotli_patch.c b/bsdiff/ss_brotli_patch.c new file mode 100644 index 0000000..6574d80 --- /dev/null +++ b/bsdiff/ss_brotli_patch.c @@ -0,0 +1,355 @@ +/* + * libtota + * + * Copyright (c) 2022 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fota_log.h" + +#define PF_OK 0 +#define PF_ERROR_OPEN_FILE 1 +#define PF_ERROR_MMAP 2 +#define PF_ERROR_INVALID_PATCH_FILE 3 +#define PF_ERROR_DECOMPRESSION 4 + +#define BUFF_IN_LEN 4096 +#define BUFF_OUT_LEN 4096 +#define SSINT_LEN 8 +#define BLOCK_COUNT_REPORT 5000 + +const char SSDIFF_MAGIC[] = "SSDIFF40"; + +struct bs_data { + int src_fd, dest_fd, patch_fd; + void *src_ptr, *dest_ptr, *patch_ptr; + size_t src_len, dest_len, patch_len; + unsigned char buff_in[BUFF_IN_LEN]; + unsigned char buff_out[BUFF_IN_LEN]; + uint8_t *dest_pos; + uint8_t *src_pos; + size_t available_in, available_out; + const uint8_t *compressed_pos; + uint8_t *decompressed_pos; + size_t total_size; + BrotliDecoderState *bstate; +}; + +static void free_data(struct bs_data *data) +{ + if (data == NULL) + return; + + if (data->src_ptr) munmap(data->src_ptr, data->src_len); + if (data->dest_ptr) munmap(data->dest_ptr, data->dest_len); + if (data->patch_ptr) munmap(data->patch_ptr, data->patch_len); + + if (data->src_fd) close(data->src_fd); + if (data->patch_fd) close(data->patch_fd); + if (data->dest_fd) close(data->dest_fd); +} + +static int open_file(char *file_name, int mode) +{ + assert(file_name); + int fd = open(file_name, mode, S_IWUSR | S_IRUSR); + if (fd < 0) + LOGE("Open file %s error: %m (%d)\n", file_name, errno); + return fd; +} + +static size_t get_file_len(int fd) +{ + assert(fd >= 0); + size_t result = lseek(fd, 0, SEEK_END); + lseek(fd, 0, SEEK_SET); + return result; +} + + +static size_t decompress_bytes(struct bs_data *data, size_t keep_offset) +{ + assert(data); + if (keep_offset > 0) { + memcpy(data->buff_out, data->buff_out + sizeof(data->buff_out) - keep_offset, keep_offset); + } + data->decompressed_pos = data->buff_out + keep_offset; + data->available_out = sizeof(data->buff_out) - keep_offset; + + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + + result = BrotliDecoderDecompressStream(data->bstate, + &data->available_in, + &data->compressed_pos, + &data->available_out, + &data->decompressed_pos, + &data->total_size); + + if (result == BROTLI_DECODER_RESULT_ERROR) { + LOGE("Decoder error\n"); + return PF_ERROR_DECOMPRESSION; + } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + LOGE("Invalid source file\n"); + return PF_ERROR_DECOMPRESSION; + } + + return PF_OK; +} + +static int open_files(struct bs_data *data, char *source_file, size_t source_size, char *dest_file, size_t dest_size, char *patch_file) +{ + assert(data); + assert(source_file); + assert(dest_file); + assert(patch_file); + + data->src_fd = open_file(source_file, O_RDONLY); + data->patch_fd = open_file(patch_file, O_RDONLY); + data->dest_fd = open_file(dest_file, O_RDWR); + if (data->src_fd < 0 || + data->patch_fd < 0 || + data->dest_fd < 0) + return PF_ERROR_OPEN_FILE; + + data->src_len = source_size; + data->patch_len = get_file_len(data->patch_fd); + data->dest_len = dest_size; + + data->src_ptr = mmap(NULL, data->src_len, PROT_READ, MAP_PRIVATE, data->src_fd, 0); + if (data->src_ptr == MAP_FAILED) { + LOGE("mmap source file error: %m (%d)", errno); + return PF_ERROR_MMAP; + } + + data->patch_ptr = mmap(NULL, data->patch_len, PROT_READ, MAP_PRIVATE, data->patch_fd, 0); + if (data->patch_ptr == MAP_FAILED) { + LOGE("mmap patch file error: %m (%d)", errno); + return PF_ERROR_MMAP; + } + + data->dest_ptr = mmap(NULL, data->dest_len, PROT_WRITE, MAP_SHARED, data->dest_fd, 0); + if (data->dest_ptr == MAP_FAILED) { + LOGE("mmap destination error: %m (%d)\n", errno); + return PF_ERROR_MMAP; + } + + data->compressed_pos = data->patch_ptr; + data->available_in = data->patch_len; + + return PF_OK; +} + +static void init_data(struct bs_data *data) +{ + assert(data); + + data->src_fd = -1; + data->patch_fd = -1; + data->dest_fd = -1; + data->src_ptr = NULL; + data->dest_ptr = NULL; + data->patch_ptr = NULL; + data->src_len = 0; + data->dest_len = 0; + data->patch_len = 0; + data->available_in = 0; + data->compressed_pos = 0; + data->available_out = 0; + data->decompressed_pos = 0; + data->bstate = BrotliDecoderCreateInstance(NULL, NULL, NULL); +} + +static int64_t parse_ssint(unsigned char *buff) +{ + assert(buff); + /* + * From bsdiff 4.0 documentation: + * + * INTEGER type: + * + * offset size data type value + * 0 1 byte x0 + * 1 1 byte x1 + * 2 1 byte x2 + * 3 1 byte x3 + * 4 1 byte x4 + * 5 1 byte x5 + * 6 1 byte x6 + * 7 1 byte x7 + 128 * s + * + * The values x0, x2, x2, x3, x4, x5, x6 are between 0 and 255 (inclusive). + * The value x7 is between 0 and 127 (inclusive). The value s is 0 or 1. + * + * The INTEGER is parsed as: + * (x0 + x1 * 256 + x2 * 256^2 + x3 * 256^3 + x4 * 256^4 + + * x5 * 256^5 + x6 * 256^6 + x7 * 256^7) * (-1)^s + * + * (In other words, an INTEGER is a 64-byte signed integer in sign-magnitude + * format, stored in little-endian byte order.) + */ + int64_t result = *(int64_t*)buff & 0x7fffffff; + if ((buff[7] & 0x80) != 0) + result = -result; + + return result; +} + +int read_header(struct bs_data *data, uint8_t **buff_out_pos) +{ + assert(data); + assert(buff_out_pos); + + *buff_out_pos = data->buff_out; + + if (*buff_out_pos + sizeof(SSDIFF_MAGIC) > data->decompressed_pos || + memcmp(data->buff_out, SSDIFF_MAGIC, sizeof(SSDIFF_MAGIC) - 1) != 0) { + LOGE("Invalid patch file\n"); + return PF_ERROR_INVALID_PATCH_FILE; + } else { + LOGL(LOG_SSENGINE, "Looks like SSDIFF\n"); + } + + *buff_out_pos += sizeof(SSDIFF_MAGIC) - 1; + + if (*buff_out_pos + SSINT_LEN > data->decompressed_pos) { + decompress_bytes(data, data->decompressed_pos - *buff_out_pos); + *buff_out_pos = data->buff_out; + } + + size_t target_size = parse_ssint(*buff_out_pos); + LOGL(LOG_SSENGINE, "target_size: 0x%lx (%ld)\n", target_size, target_size); + + if (target_size != data->dest_len) { + LOGE("Declared target size differs from that read from the patch\n"); + return PF_ERROR_INVALID_PATCH_FILE; + } + + *buff_out_pos += SSINT_LEN; + + return PF_OK; +} + +int apply_patch_brotli(char *source_file, size_t source_size, char *dest_file, size_t dest_size, char *patch_file) +{ + assert(source_file); + assert(dest_file); + assert(patch_file); + + int result; + uint64_t blocks = 0; + struct bs_data data; + + init_data(&data); + + if ((result = open_files(&data, source_file, source_size, dest_file, dest_size, patch_file)) != PF_OK) + goto exit; + + if ((result = decompress_bytes(&data, 0)) != PF_OK) + goto exit; + + uint8_t *buff_out_pos; + + if ((result = read_header(&data, &buff_out_pos)) != PF_OK) + goto exit; + + uint64_t total_write = 0; + + while (total_write < data.dest_len) { + /* + * Make sure we can read the block header + */ + if (buff_out_pos + 4*8 > data.decompressed_pos) { + if ((result = decompress_bytes(&data, data.decompressed_pos - buff_out_pos)) != PF_OK) + goto exit; + buff_out_pos = data.buff_out; + } + + /* + * Read the block header + */ + int64_t diff_len = parse_ssint(buff_out_pos+0*8); + int64_t extra_len = parse_ssint(buff_out_pos+1*8); + int64_t old_pos = parse_ssint(buff_out_pos+2*8); + int64_t new_pos = parse_ssint(buff_out_pos+3*8); + buff_out_pos += 4*8; + + /* + * Prepare pointers + */ + data.dest_pos = data.dest_ptr + new_pos; + data.src_pos = data.src_ptr + old_pos; + /* + * Read diff data + */ + int64_t write = 0; + while (write < diff_len) { + if (buff_out_pos >= data.decompressed_pos) { + if ((result = decompress_bytes(&data, 0)) != PF_OK) + goto exit; + buff_out_pos = data.buff_out; + } + while (write < diff_len && buff_out_pos < data.decompressed_pos) { + *data.dest_pos = *(uint8_t*)(data.src_pos) + *(uint8_t*)buff_out_pos; + data.dest_pos++; + data.src_pos++; + buff_out_pos++; + write++; + } + } + total_write += write; + /* + * Read extra data + */ + write = 0; + while (write < extra_len) { + if (buff_out_pos >= data.decompressed_pos) { + if ((result = decompress_bytes(&data, 0)) != PF_OK) + goto exit; + buff_out_pos = data.buff_out; + } + int64_t chunk_size = extra_len - write; + if (buff_out_pos + chunk_size > data.decompressed_pos) { + chunk_size = data.decompressed_pos - buff_out_pos; + } + memcpy(data.dest_pos, buff_out_pos, chunk_size); + data.dest_pos += chunk_size; + buff_out_pos += chunk_size; + write += chunk_size; + } + total_write += write; + + blocks++; + if (blocks % BLOCK_COUNT_REPORT == 0) { + printf("Number of processed patch blocks: %lld\n", blocks); + } + } + + result = PF_OK; + +exit: + printf("Total processed blocks: %lld\n", blocks); + free_data(&data); + return result; +} diff --git a/bsdiff/ss_brotli_patch.h b/bsdiff/ss_brotli_patch.h new file mode 100644 index 0000000..47694b9 --- /dev/null +++ b/bsdiff/ss_brotli_patch.h @@ -0,0 +1,22 @@ +/* + * libtota + * + * Copyright (c) 2022 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +extern int apply_patch_brotli(char *source_file, size_t source_size, char *dest_file, size_t dest_size, char *patch_file); diff --git a/bsdiff/ss_bsdiff.c b/bsdiff/ss_bsdiff.c new file mode 100644 index 0000000..76fbe41 --- /dev/null +++ b/bsdiff/ss_bsdiff.c @@ -0,0 +1,805 @@ +/*- + * Copyright 2003-2005 Colin Percival + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +// This file is a nearly copy of bsdiff.c from the +// bsdiff-4.3 distribution; the primary differences being how the +// input and output data are read, data post processing, +// search function modification and the error handling. + +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include +#include + +#define SUFSORT_MOD // Change suffix sorting algorithm from Qsufsort to Divsufsort +//#define ZLIB_MOD // Change compression algorithm +#define SEARCH_RECURSIVE // Change recursion of Search function to Iteration +//#define MAX_MATCH_SIZE // define ( MAX_MATCH_SIZE or CONST_MEMORY_USAGE ) or ( none of them ); use max_match memory for old file at patch side +#define CONST_MEMORY_USAGE 16384 // patch in m+O(1); m=size of new file; use only const memory for old file at patch side; +#define PATCH_FILE_FORMAT_MOD // no accumulation of diff and extra in db and eb; immediate write; also write all 3 parts of control stmt at same time +#define MULTI_THREADING 1 // only with #define CONST_MEMORY_USAGE or #define MAX_MATCH_SIZE +#define TIME_LIMIT_CHECK 4*60*60 // After TIME_LIMIT_CHECK seconds, new diff will be created with smaller score argument +#define TEMP_PATCH_NAME "temp_patch" +#define BROTLI_COMPRESSION_QUALITY 9 +#define ITERATIONS_COMPLETED 10 // After ITERATIONS_COMPLETED iterations elapsed time will be checked. Increasing it can cause the program to run longer than expected + // (the longer the program runs the longer the iterations) + +/* Take care : +1) Use either (MAX_MATCH_SIZE or CONST_MEMORY_USAGE) or (none of both). +2.) PATCH_FILE_FORMAT_MOD may or may not be used, independently of everything else. +3.) MULTI_THREADING can be used only with (CONST_MEMORY_USAGE or MAX_MATCH_SIZE). +*/ +#ifdef TIME_LIMIT_CHECK +long long outer_count = 0; +char ts1[256]; +void get_time_stamp(void) +{ + struct timeval tv; + int sec, msec; + + gettimeofday(&tv, NULL); + sec = (int)tv.tv_sec; + msec = (int)(tv.tv_usec / 1000); + snprintf(ts1, 256, "%06d.%03d", sec % 100000, msec); +} +#endif +#ifdef SUFSORT_MOD +//supporting only 32 bit divsufsort for now. +#include +#endif + +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) + +#ifdef MULTI_THREADING + +struct data_thread { + unsigned num_threads; + off_t size_thread; + + int fd; + int rc; + u_char *old; + u_char **new; + off_t oldsize; + off_t newsize; + + saidx_t *I; + u_char buf[8]; + u_char header[16]; + u_char buf2[32]; + + off_t lenn; + off_t lenn2; + + FILE * pf; + int bz2err; + FILE * pfbz2; +}; + +enum compression_method { + CM_LZMA, + CM_BROTLI, +}; + +struct bsdiff_info { + const char *old_file; + const char *new_file; + const char *patch_file; + enum compression_method comp_method; +}; + +struct data_thread data; + +int Function(int); + + +#endif + +static off_t matchlen(u_char *old, off_t oldsize, u_char *new, off_t newsize) +{ + off_t i; + for (i = 0; (i < oldsize) && (i < newsize); i++) + if (old[i] != new[i]) + break; + return i; +} + +static off_t search(saidx_t *I, u_char *old, off_t oldsize, + u_char *new, off_t newsize, off_t st, off_t en, off_t *pos) +{ + off_t x, y; + while (en - st >= 2) { + x = st + (en - st) / 2; + if (memcmp(old + I[x], new, MIN(oldsize - I[x], newsize)) < 0) + st = x; + else + en = x; + } + + x = matchlen(old + I[st], oldsize - I[st], new, newsize); + y = matchlen(old + I[en], oldsize - I[en], new, newsize); + + if (x > y) { + *pos = I[st]; + return x; + } else { + *pos = I[en]; + return y; + } +} + +static void offtout(off_t x, u_char *buf) +{ + off_t y; + + if (x < 0) + y = -x; + else + y = x; + + buf[0] = y % 256; + y -= buf[0]; + y = y / 256; + buf[1] = y % 256; + y -= buf[1]; + y = y / 256; + buf[2] = y % 256; + y -= buf[2]; + y = y / 256; + buf[3] = y % 256; + y -= buf[3]; + y = y / 256; + buf[4] = y % 256; + y -= buf[4]; + y = y / 256; + buf[5] = y % 256; + y -= buf[5]; + y = y / 256; + buf[6] = y % 256; + y -= buf[6]; + y = y / 256; + buf[7] = y % 256; + + if (x < 0) + buf[7] |= 0x80; +} + +static off_t bulk_read(int fd, u_char * buf, off_t size) +{ + off_t bytes_read = 0; + int ret = 0; + + while(bytes_read < size) + { + ret = read(fd, buf + bytes_read, size - bytes_read); + if (ret > 0) + bytes_read += ret; + else + break; + } + return bytes_read; +} + +int create_patch(const char *old_file, const char *new_file, const char *temp_patch, int offset_oldscore) +{ + assert(old_file); + assert(new_file); + + data.num_threads = MULTI_THREADING; + data.new = (u_char **)malloc(sizeof(u_char *)*data.num_threads); + + /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if (((data.fd = open(old_file, O_RDONLY, 0)) < 0) || + ((data.oldsize = lseek(data.fd, 0, SEEK_END)) == -1) || + ((data.old = malloc(data.oldsize + 1)) == NULL) || + (lseek(data.fd, 0, SEEK_SET) != 0) || + (bulk_read(data.fd, data.old, data.oldsize) != data.oldsize) || + (close(data.fd) == -1)) + err(1, "%s", old_file); + + data.I = malloc((data.oldsize + 1) * sizeof(saidx_t)); + if (!data.I) + err(1, "Memory allocation error: %s", old_file); + divsufsort(data.old, data.I, data.oldsize); + + /* Allocate newsize+1 bytes instead of newsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if (((data.fd = open(new_file, O_RDONLY, 0)) < 0) || + ((data.newsize = lseek(data.fd, 0, SEEK_END)) == -1) || + (lseek(data.fd, 0, SEEK_SET) != 0)) + err(1, "%s", new_file); + data.size_thread = (data.newsize / data.num_threads); + + unsigned int j; + for (j = 0; j < data.num_threads; ++j) { + if (j != data.num_threads - 1) { + if (((data.new[j] = (u_char *)malloc(sizeof(u_char) * (data.size_thread + 1))) == NULL) || + (lseek(data.fd, 0, SEEK_CUR) != j * data.size_thread) || + (bulk_read(data.fd, data.new[j], data.size_thread) != data.size_thread)) + err(1, "%s", new_file); + } else { + if (((data.new[j] = (u_char *)malloc(sizeof(u_char) * (data.newsize - (j * data.size_thread) + 1))) == NULL) || + (lseek(data.fd, 0, SEEK_CUR) != j * data.size_thread) || + (bulk_read(data.fd, data.new[j], data.newsize - (j * data.size_thread)) != data.newsize - (j * data.size_thread))) + err(1, "here %s", new_file); + } + } + + if ((close(data.fd) == -1)) + err(1, "%s", new_file); + + /* Create the patch file */ + if ((data.pf = fopen(temp_patch, "w")) == NULL) + err(1, "%s", temp_patch); + + /* Header is + 0 8 "BSDIFF40" + 8 8 length of bzip2ed ctrl block + 16 8 length of bzip2ed diff block + 24 8 length of new file */ + /* File is + 0 32 Header + 32 ?? Bzip2ed ctrl block + ?? ?? Bzip2ed diff block + ?? ?? Bzip2ed extra block */ + memcpy(data.header, "SSDIFF40", 8); + + offtout(data.newsize, data.header + 8); + if (fwrite(data.header, 16, 1, data.pf) != 1) + err(1, "fwrite(%s)", temp_patch); + + /* Compute the differences, writing ctrl as we go */ + data.pfbz2 = data.pf; + //if ((data.pfbz2 = BZ2_bzWriteOpen(&data.bz2err, data.pf, 9, 0, 0)) == NULL) + //errx(1, "BZ2_bzWriteOpen, bz2err = %d", data.bz2err); + + + //BZ2_bzWriteClose(&data.bz2err, data.pfbz2, 0, NULL, NULL); + //if (data.bz2err != BZ_OK) + //errx(1, "BZ2_bzWriteClose, bz2err = %d", data.bz2err); + + int ret = Function(offset_oldscore); +#ifdef TIME_LIMIT_CHECK + if (ret != 0) { + printf("bsdiff fails to create delta with offset score %d\n", offset_oldscore); + printf("Old: [%s] -> New: [%s]\n", old_file, new_file); + } +#endif + /* Seek to the beginning, write the header, and close the file */ + if (fseeko(data.pf, 0, SEEK_SET)) + err(1, "fseeko"); + + if (fwrite(data.header, 16, 1, data.pf) != 1) + err(1, "fwrite(%s)", temp_patch); + if (fclose(data.pf)) + err(1, "fclose"); + /* Free the memory we used */ + free(data.I); + free(data.old); + free(data.new); + + return ret; +} + +int Function(int offset_oldscore) +{ + unsigned int thread_num = 0; + off_t end; + off_t scan = 0; + end = data.newsize; + int t1 = 0, t2 = 0; + get_time_stamp(); //total time capturing + t1 = atoi(ts1); + +#ifdef PATCH_FILE_FORMAT_MOD + u_char* db; + u_char* eb; + off_t dblen; + off_t eblen; +#endif + + off_t pos; + off_t len; + off_t lastscan; + off_t lastpos; + off_t lastoffset; + off_t oldscore; + off_t scsc; + off_t s; + off_t Sf; + off_t lenf; + off_t Sb; + off_t lenb; + off_t overlap; + off_t Ss; + off_t lens; + off_t i; + + pos = 0; + len = 0; + lastscan = 0; + lastpos = 0; + lastoffset = 0; + while (scan < end) { + oldscore = 0; + size_t prev_len; + uint64_t prev_pos, prev_oldscore; + int num_less_than_eight = 0; + for (scsc = scan += len; scan < end; scan++) { + prev_len = len; + prev_oldscore = oldscore; + prev_pos = pos; + len = search(data.I, data.old, data.oldsize, data.new[thread_num] + scan, end - scan, + len, data.oldsize, &pos); // Passing parameter as len instead of 0 for ramdisk.img etc taking long time + + for (; scsc < scan + len; scsc++) + if ((scsc + lastoffset < data.oldsize) && + (data.old[scsc + lastoffset] == data.new[thread_num][scsc])) + oldscore++; +#ifdef TIME_LIMIT_CHECK + if (offset_oldscore > 4) // when offset_oldscore is 4 and less we have to make sure diff is created no mater what, so we can't timeout + outer_count++; + if (outer_count > ITERATIONS_COMPLETED) { + outer_count = 0; + get_time_stamp(); //total time capturing + t2 = atoi(ts1); + //printf("\ntime diff = %d\n", (t2 - t1)); + if ((t2 - t1) > TIME_LIMIT_CHECK) + return 1; + } +#endif + if (((len == oldscore) && (len != 0)) || + (len > oldscore + offset_oldscore)) + break; + + if ((scan + lastoffset < data.oldsize) && + (data.old[scan + lastoffset] == data.new[thread_num][scan])) + oldscore--; + + /* + * Modification created by Thieu Le + * which speeds up patch generation time in situations + * where large blocks of data differ by less than 8 bytes. + * https://android.googlesource.com/platform/external/bsdiff/+/d172820cb8b4513478f0db5546d6e4d388adc1a7 + */ + const size_t fuzz = 8; + if (prev_len - fuzz <= len && len <= prev_len && + prev_oldscore - fuzz <= oldscore && + prev_pos <= pos && pos <= prev_pos + fuzz && + oldscore <= len && len <= oldscore + fuzz) { + num_less_than_eight++; + } else { + num_less_than_eight = 0; + } + if (num_less_than_eight > 100) break; + + }; + if ((len != oldscore) || (scan == end)) { + s = 0; + Sf = 0; + lenf = 0; + for (i = 0; (lastscan + i < scan) && (lastpos + i < data.oldsize); ) { + if (data.old[lastpos + i] == data.new[thread_num][lastscan + i]) + s++; + i++; + if (s * 2 - i > Sf * 2 - lenf) { + Sf = s; + lenf = i; + }; + }; + + lenb = 0; + if (scan < end) { + s = 0; + Sb = 0; + for (i = 1; (scan >= lastscan + i) && (pos >= i); i++) { + if (data.old[pos - i] == data.new[thread_num][scan - i]) + s++; + if (s * 2 - i > Sb * 2 - lenb) { + Sb = s; + lenb = i; + }; + }; + }; + + if (lastscan + lenf > scan - lenb) { + overlap = (lastscan + lenf) - (scan - lenb); + s = 0; + Ss = 0; + lens = 0; + for (i = 0; i < overlap; i++) { + if (data.new[thread_num][lastscan + lenf - overlap + i] == + data.old[lastpos + lenf - overlap + i]) + s++; + if (data.new[thread_num][scan - lenb + i] == + data.old[pos - lenb + i]) + s--; + if (s > Ss) { + Ss = s; + lens = i + 1; + }; + }; + + lenf += lens - overlap; + lenb -= lens; + }; + + if (((db = malloc(lenf + 1)) == NULL) || + ((eb = malloc((scan - lenb) - (lastscan + lenf) + 1)) == NULL)) + err(1, NULL); + + for (i = 0; i < lenf; i++) + db[i] = data.new[thread_num][lastscan + i] - data.old[lastpos + i]; + for (i = 0; i < (scan - lenb) - (lastscan + lenf); i++) + eb[i] = data.new[thread_num][lastscan + lenf + i]; + dblen = lenf; + eblen = (scan - lenb) - (lastscan + lenf); + offtout(lenf, data.buf2); + offtout((scan - lenb) - (lastscan + lenf), data.buf2 + 8); + offtout(lastpos, data.buf2 + 16); + offtout((data.size_thread * thread_num) + lastscan, data.buf2 + 24); + fwrite(data.buf2, 1, 32, data.pf); + //if (data.bz2err != BZ_OK) + //errx(1, "fwrite, bz2err = %d", data.bz2err); + + fwrite(db, 1, dblen, data.pf); + //if (data.bz2err != BZ_OK) + //errx(1, "fwrite, bz2err = %d", data.bz2err); + + fwrite(eb, 1, eblen, data.pf); + //if (data.bz2err != BZ_OK) + //errx(1, "fwrite, bz2err = %d", data.bz2err); + + free(db); + free(eb); + + lastscan = scan - lenb; + lastpos = pos - lenb; + lastoffset = pos - scan; + }; + }; + return 0; +} + +const char *kCantReadMessage = "Can not read input file"; +const char *kCantWriteMessage = "Can not write output file"; +const char *kCantAllocateMessage = "Can not allocate memory"; +const char *kDataErrorMessage = "Data error"; + +static void *SzAlloc(void *p, size_t size) +{ + p = p; + return MyAlloc(size); +} +static void SzFree(void *p, void *address) +{ + p = p; + MyFree(address); +} +static ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +int PrintError(char *buffer, const char *message, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError: %s\n", message); + return 1; +} + +int PrintErrorNumber(char *buffer, SRes val, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError code: %x\n", (unsigned)val); + return 1; +} + +int PrintUserError(char *buffer, int buf_size) +{ + return PrintError(buffer, "Incorrect command", buf_size); +} + +#define IN_BUF_SIZE (1 << 16) +#define OUT_BUF_SIZE (1 << 16) + + +static SRes lzma_encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, char *rs) +{ + CLzmaEncHandle enc; + SRes res; + CLzmaEncProps props; + + rs = rs; + + enc = LzmaEnc_Create(&g_Alloc); + if (enc == 0) + return SZ_ERROR_MEM; + + LzmaEncProps_Init(&props); + res = LzmaEnc_SetProps(enc, &props); + + if (res == SZ_OK) { + Byte header[LZMA_PROPS_SIZE + 8]; + size_t headerSize = LZMA_PROPS_SIZE; + int i; + + res = LzmaEnc_WriteProperties(enc, header, &headerSize); + for (i = 0; i < 8; i++) + header[headerSize++] = (Byte)(fileSize >> (8 * i)); + if (outStream->Write(outStream, header, headerSize) != headerSize) + res = SZ_ERROR_WRITE; + else { + if (res == SZ_OK) + res = LzmaEnc_Encode(enc, outStream, inStream, NULL, &g_Alloc, &g_Alloc); + } + } + LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); + return res; +} + +int lzma_compress(const char *input_file, const char *output_file, char *rs, int rs_size) +{ + assert(input_file); + assert(output_file); + assert(rs); + + CFileSeqInStream inStream; + CFileOutStream outStream; + int res; + + FileSeqInStream_CreateVTable(&inStream); + File_Construct(&inStream.file); + + FileOutStream_CreateVTable(&outStream); + File_Construct(&outStream.file); + + size_t t4 = sizeof(UInt32); + size_t t8 = sizeof(UInt64); + if (t4 != 4 || t8 != 8) + return PrintError(rs, "Incorrect UInt32 or UInt64", rs_size); + + if (InFile_Open(&inStream.file, input_file) != 0) + return PrintError(rs, "Can not open input file", rs_size); + + + if (OutFile_Open(&outStream.file, output_file) != 0) + return PrintError(rs, "Can not open output file", rs_size); + + + UInt64 fileSize; + File_GetLength(&inStream.file, &fileSize); + res = lzma_encode(&outStream.s, &inStream.s, fileSize, rs); + + File_Close(&outStream.file); + File_Close(&inStream.file); + + if (res != SZ_OK) { + if (res == SZ_ERROR_MEM) + return PrintError(rs, kCantAllocateMessage, rs_size); + else if (res == SZ_ERROR_DATA) + return PrintError(rs, kDataErrorMessage, rs_size); + else if (res == SZ_ERROR_WRITE) + return PrintError(rs, kCantWriteMessage, rs_size); + else if (res == SZ_ERROR_READ) + return PrintError(rs, kCantReadMessage, rs_size); + return PrintErrorNumber(rs, res, rs_size); + } + return 0; +} + +int brotli_compress_internal(int input_fd, int output_fd, int quality) +{ + int res = -1; + size_t input_size = lseek(input_fd, 0, SEEK_END); + lseek(input_fd, 0, SEEK_SET); + void *input_file_ptr = mmap(NULL, input_size, PROT_READ, MAP_PRIVATE, input_fd, 0); + + if (input_file_ptr == MAP_FAILED) { + printf("Can not mmap input file: %d - %m\n", errno); + goto exit; + } + + BrotliEncoderState *bstate = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (bstate == 0) { + printf("Can not create BrotliEncoder instance\n"); + goto exit; + } + size_t max_output_size = BrotliEncoderMaxCompressedSize(input_size); + + if (max_output_size == 0) { + printf("Brotli engine error\n"); + goto exit; + } + + if (ftruncate(output_fd, max_output_size) == -1) { + printf("Can not truncate output file: %d - %m\n", errno); + goto exit; + } + + void *output_file_ptr = mmap(NULL, max_output_size, PROT_WRITE, MAP_SHARED, output_fd, 0); + if (output_file_ptr == MAP_FAILED) { + printf("Can not mmap output file: %d - %m\n", errno); + goto exit; + } + + if(!BrotliEncoderCompress(quality, + BROTLI_DEFAULT_WINDOW, + BROTLI_DEFAULT_MODE, + input_size, + input_file_ptr, + &max_output_size, + output_file_ptr)) { + printf("Compression error\n"); + goto exit; + } + if (ftruncate(output_fd, max_output_size) == -1) { + printf("Can not truncate output file after compression: %d - %m\n", errno); + goto exit; + } + + res = 0; +exit: + if (input_file_ptr) + munmap(input_file_ptr, input_size); + if (output_file_ptr) + munmap(output_file_ptr, max_output_size); + + return res; +} + +int brotli_compress(const char *input_file, const char *output_file, int quality) +{ + assert(input_file); + assert(output_file); + int res = -1; + + int input_fd = open(input_file, O_RDONLY); + if (input_fd < 0) { + printf("Can not open file: %s for read\n", input_file); + return res; + } + int output_fd = open(output_file, O_RDWR | O_CREAT, S_IWUSR | S_IRUSR); + if (output_fd < 0) { + printf("Can not open file: %s for write (%d: %m)\n", output_file, errno); + close(input_fd); + return res; + } + + res = brotli_compress_internal(input_fd, output_fd, quality); + + close(input_fd); + close(output_fd); + + return res; +} + +void print_help(const char *arg0) +{ + assert(arg0); + errx(1, "ss_bsdiff Version 5.0\nUsage: %s [-c ] oldfile newfile patchfile\n", arg0); +} + +int parse_args(struct bsdiff_info *info, int argc, char *argv[]) +{ + assert(info); + assert(argv); + + info->comp_method = CM_LZMA; // default compression method + + struct option long_options[] = { + {"compression", optional_argument, NULL, 'c'}, + {0, 0 , 0, 0} + }; + + int opt; + + while ((opt = getopt_long(argc, argv, "c:", long_options, NULL)) != -1) { + switch (opt) { + case 'c': + if (strcmp("lzma", optarg) == 0) + info->comp_method = CM_LZMA; + else if (strcmp("brotli", optarg) == 0) + info->comp_method = CM_BROTLI; + else { + err(1, "Unknown compression method: %s", optarg); + return -1; + } + } + } + + if (optind + 2 >= argc) { + err(1, "Not enough parameters"); + print_help(argv[0]); + return -1; + } + + info->old_file = argv[optind]; + info->new_file = argv[optind+1]; + info->patch_file = argv[optind+2]; + + return 0; +} + +int MY_CDECL main(int argc, char *argv[]) +{ + char rs[800] = { 0 }; + + struct bsdiff_info info; + if (parse_args(&info, argc, argv) != 0) + return 1; + + int ret = create_patch(info.old_file, info.new_file, TEMP_PATCH_NAME, 8); +#ifdef TIME_LIMIT_CHECK + if (ret != 0) { + /* + * Creating a patch with an offset score equal to 8 may take too long + * Therefore after a certain amount of time, patch creation is aborted + * and we run again with the score equal to 4. + */ + int score = 4; + printf("Trying with offset score %d\n", score); + ret = create_patch(info.old_file, info.new_file, TEMP_PATCH_NAME, score); + } + if (ret != 0) { + if (remove(TEMP_PATCH_NAME) < 0) + printf("Failed to remove %s\n", TEMP_PATCH_NAME); + err(1, "bsdiff fails to create delta within timelimit"); + } +#endif + int res = 0; + switch(info.comp_method) { + case CM_LZMA: + res = lzma_compress(TEMP_PATCH_NAME, info.patch_file, rs, sizeof(rs)); + break; + case CM_BROTLI: + res = brotli_compress(TEMP_PATCH_NAME, info.patch_file, BROTLI_COMPRESSION_QUALITY); + break; + default: + printf("Unknown compression method\n"); + res = -1; + break; + } + + if (remove(TEMP_PATCH_NAME) < 0) + printf("Failed to remove %s\n", TEMP_PATCH_NAME); + fputs(rs, stdout); + return res; +} diff --git a/bsdiff/ss_bspatch.c b/bsdiff/ss_bspatch.c new file mode 100644 index 0000000..8b6117f --- /dev/null +++ b/bsdiff/ss_bspatch.c @@ -0,0 +1,155 @@ +/*- + * Copyright 2003-2005 Colin Percival + * Copyright 2012 Matthew Endsley + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * Modifications are made in reimplementing suffix sort array generation + * and how the data is read and written to.Iterative part replaced the + * recursive implementation to avoid buffer overflow problems + */ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include + +#include +#include +#include + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include + +#include "ss_bspatch_common.h" + +const char *kCantReadMessage = "Can not read input file"; +const char *kCantWriteMessage = "Can not write output file"; +const char *kCantAllocateMessage = "Can not allocate memory"; +const char *kDataErrorMessage = "Data error"; + +int PrintError(char *buffer, const char *message, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError: %s\n", message); + return 1; +} + +int PrintErrorNumber(char *buffer, SRes val, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError code: %x\n", (unsigned)val); + return 1; +} + +int PrintUserError(char *buffer, int buf_size) +{ + return PrintError(buffer, "Incorrect command", buf_size); +} + +int main2(int numArgs, const char *args[], char *rs, int rs_size) +{ + CFileSeqInStream inStream; + CFileOutStream outStream; + int res, fd; + int encodeMode; + unsigned char *buf_res = NULL; + unsigned char *new_data; + ssize_t new_size; + + FileSeqInStream_CreateVTable(&inStream); + File_Construct(&inStream.file); + + FileOutStream_CreateVTable(&outStream); + //File_Construct(&outStream.file); + + encodeMode = 0; + + size_t t4 = sizeof(UInt32); + size_t t8 = sizeof(UInt64); + if (t4 != 4 || t8 != 8) + return PrintError(rs, "Incorrect UInt32 or UInt64", rs_size); + + if (InFile_Open(&inStream.file, args[3]) != 0) + return PrintError(rs, "Can not open input file", rs_size); + + if (encodeMode) { + UInt64 fileSize; + File_GetLength(&inStream.file, &fileSize); + //res = Encode(&outStream.s, &inStream.s, fileSize, rs); + } else { + UInt64 unpackSize, i; + CLzmaDec state; + unsigned char header[LZMA_PROPS_SIZE + 8]; + RINOK(SeqInStream_Read(&inStream.s, header, sizeof(header))); + unpackSize = 0; + for (i = 0; i < 8; i++) + unpackSize += (UInt64)header[LZMA_PROPS_SIZE + i] << (i * 8); + buf_res = (unsigned char *)malloc(unpackSize); + if (!buf_res) + return PrintError(rs, "Failed to allocate memory", rs_size); + memset(buf_res, 0x0, unpackSize); + LzmaDec_Construct(&state); + RINOK(LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc)); + res = Decode2(&state, &outStream.s, &inStream.s, &unpackSize, buf_res); + LzmaDec_Free(&state, &g_Alloc); + File_Close(&inStream.file); + if (apply_patch(args[1], buf_res, &new_data, &new_size) != 0) { + if (new_data) + free(new_data); + return 1; + } + if (((fd = open(args[2], O_CREAT | O_TRUNC | O_WRONLY, 0666)) < 0) || + (write(fd, new_data, new_size) != new_size) || (close(fd) == -1)) + err(1, "%s", args[2]); + if (res != SZ_OK) { + free(new_data); + free(buf_res); + if (res == SZ_ERROR_MEM) + return PrintError(rs, kCantAllocateMessage, rs_size); + else if (res == SZ_ERROR_DATA) + return PrintError(rs, kDataErrorMessage, rs_size); + else if (res == SZ_ERROR_WRITE) + return PrintError(rs, kCantWriteMessage, rs_size); + else if (res == SZ_ERROR_READ) + return PrintError(rs, kCantReadMessage, rs_size); + return PrintErrorNumber(rs, res, rs_size); + } + free(new_data); + free(buf_res); + return 0; + } + return 1; +} + +int main(int numArgs, const char *args[]) +{ + char rs[800] = { 0 }; + if (numArgs != 4) + errx(1, "ss_bspatch Version 1.0\nUsage: ss_bspatch oldfile newfile patchfile\n"); + int res = main2(numArgs, args, rs, sizeof(rs)); + fputs(rs, stdout); + return res; +} diff --git a/bsdiff/ss_bspatch_common.c b/bsdiff/ss_bspatch_common.c new file mode 100644 index 0000000..e7bd45e --- /dev/null +++ b/bsdiff/ss_bspatch_common.c @@ -0,0 +1,291 @@ +/*- + * Copyright 2003-2005 Colin Percival + * Copyright 2012 Matthew Endsley + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * Modifications are made in reimplementing suffix sort array generation + * and how the data is read and written to.Iterative part replaced the + * recursive implementation to avoid buffer overflow problems + */ +//#define ZLIB_MOD //not stable yet. +//#define MAX_MATCH_SIZE // define ( MAX_MATCH_SIZE or CONST_MEMORY_USAGE ) or ( none of them ) +#define CONST_MEMORY_USAGE (64*1024) //tests show smallest time when using 64 kb +#define PATCH_FILE_FORMAT_MOD +#define BSDIFF_HEADER "BSDIFF40" +#define SSDIFF_HEADER "SSDIFF40" +//#define MULTI_THREADING +#include +#include +#include + +#include +#include +#include + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include + +static void *SzAlloc(void *p, size_t size) +{ + p = p; + return MyAlloc(size); +} + +static void SzFree(void *p, void *address) +{ + p = p; + MyFree(address); +} +ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +static off_t offtin(u_char *buf) +{ + off_t y; + + y = buf[7] & 0x7F; + y = y * 256; + y += buf[6]; + y = y * 256; + y += buf[5]; + y = y * 256; + y += buf[4]; + y = y * 256; + y += buf[3]; + y = y * 256; + y += buf[2]; + y = y * 256; + y += buf[1]; + y = y * 256; + y += buf[0]; + + if (buf[7] & 0x80) + y = -y; + + return y; +} + +#define IN_BUF_SIZE (1 << 16) +#define OUT_BUF_SIZE (1 << 16) + +SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, + UInt64 *unpackSize, unsigned char *dec_data) +{ + int thereIsSize = (*unpackSize != (UInt64)(Int64) - 1); + UInt64 offset = 0; + Byte inBuf[IN_BUF_SIZE]; + Byte outBuf[OUT_BUF_SIZE]; + size_t inPos = 0, inSize = 0, outPos = 0; + + LzmaDec_Init(state); + + offset = 0; + + for (;;) { + if (inPos == inSize) { + inSize = IN_BUF_SIZE; + RINOK(inStream->Read(inStream, inBuf, &inSize)); + inPos = 0; + } + + SRes res; + SizeT inProcessed = inSize - inPos; + SizeT outProcessed = OUT_BUF_SIZE - outPos; + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + ELzmaStatus status; + + if (thereIsSize && outProcessed > *unpackSize) { + outProcessed = (SizeT) * unpackSize; + finishMode = LZMA_FINISH_END; + } + + res = LzmaDec_DecodeToBuf(state, outBuf + outPos, &outProcessed, + inBuf + inPos, &inProcessed, finishMode, &status); + inPos += inProcessed; + outPos += outProcessed; + *unpackSize -= outProcessed; + memcpy(dec_data + offset, outBuf, outProcessed); + offset += outProcessed; + + outPos = 0; + + if ((res != SZ_OK) || (thereIsSize && *unpackSize == 0)) + return res; + + if (inProcessed == 0 && outProcessed == 0) { + if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK) + return SZ_ERROR_DATA; + return res; + } + } +} + +int apply_patch(const char *oldfile, unsigned char *patch_buffer, unsigned char **dest_buf, ssize_t *dest_size) +{ + int fd = -1, result = 0; + off_t oldsize, newsize; + u_char header[16], buf[8]; + u_char *old = NULL; + off_t oldpos, newpos; + off_t ctrl[4]; /////////////////////////////////////THREAD + off_t total_write; /////////////////////////////////////////THREAD + off_t j; + off_t memory_usage = CONST_MEMORY_USAGE; + off_t match_size; + off_t patch_buffer_offset = 0; + bool flag; + + /* + File format: + 0 8 "BSDIFF40" + 8 8 X + 16 8 Y + 24 8 sizeof(newfile) + 32 X bzip2(control block) + 32+X Y bzip2(diff block) + 32+X+Y ??? bzip2(extra block) + with control block a set of triples (x,y,z) meaning "add x bytes + from oldfile to x bytes from the diff block; copy y bytes from the + extra block; seek forwards in oldfile by z bytes". + */ + // Read header + if (patch_buffer) + memcpy(header, patch_buffer, 16); + else { + printf("%s().%d Corrupt decoded patch buffer\n", __FUNCTION__, __LINE__); + return 1; + } + + /* Check for appropriate magic */ + if (memcmp(header, BSDIFF_HEADER, 8) != 0 && memcmp(header, SSDIFF_HEADER, 8) != 0) { + printf("%s().%d Patch buffer header corrupt\n", __FUNCTION__, __LINE__); + return 1; + } + + /* Read lengths from header */ + newsize = offtin(header + 8); + + if ((newsize < 0)) { + printf("%s().%d Patch buffer corrupt\n", __FUNCTION__, __LINE__); + return 1; + } + + /* Cset patch_buffer_offset at the right place */ + patch_buffer_offset += 16; + + if (((fd = open(oldfile, O_RDONLY, 0)) < 0) || + ((oldsize = lseek(fd, 0, SEEK_END)) == -1) || + ((old = malloc(memory_usage + 1)) == NULL) || + (lseek(fd, 0, SEEK_SET) != 0)) { + printf("Corruption in old file %s\n", oldfile); + result = 1; + goto Cleanup; + } + + if ((*dest_buf = malloc(newsize + 1)) == NULL) { + printf("Corruption in old file %s\n", oldfile); + result = 1; + goto Cleanup; + } + oldpos = 0; + newpos = 0; + + total_write = 0; + + while (total_write != newsize) { + /* Read control data */ + for (j = 0; j <= 3; j++) { + memcpy(buf, patch_buffer + patch_buffer_offset, 8); + patch_buffer_offset += 8; + ctrl[j] = offtin(buf); + }; + + total_write += (ctrl[0] + ctrl[1]); + newpos = ctrl[3]; + oldpos = ctrl[2]; + + ////////////////////////////////////////////////////////////////////////////////// + flag = true; + match_size = ctrl[0]; + while (flag == true) { + if (match_size <= memory_usage) { + if (pread(fd, old, match_size, oldpos) != match_size) { + printf("Corruption in old file %s\n", oldfile); + result = 1; + goto Cleanup; + } + if (newpos + match_size > newsize) { + printf("%s().%d Corrupt patch\n", __FUNCTION__, __LINE__); + result = 1; + goto Cleanup; + } + memcpy((*dest_buf) + newpos, patch_buffer + patch_buffer_offset, match_size); + patch_buffer_offset += match_size; + for (j = 0; j < match_size; j++) + (*dest_buf)[newpos + j] += old[j]; + newpos += match_size; + flag = false; + } else { + if (pread(fd, old, memory_usage, oldpos) != memory_usage) { + printf("%s().%d Corruption in old file %s\n", __FUNCTION__, __LINE__ , oldfile); + result = 1; + goto Cleanup; + } + if (newpos + memory_usage > newsize) { + printf("%s().%d Corrupt patch\n", __FUNCTION__, __LINE__); + result = 1; + goto Cleanup; + } + memcpy((*dest_buf) + newpos, patch_buffer + patch_buffer_offset, memory_usage); + patch_buffer_offset += memory_usage; + for (j = 0; j < memory_usage; j++) + (*dest_buf)[newpos + j] += old[j]; + match_size -= memory_usage; + oldpos += memory_usage; + newpos += memory_usage; + } + } + + //////////////////////////////////////////////////////////////////////////////////////// + /* Sanity-check */ + if (newpos + ctrl[1] > newsize) { + printf("%s().%d Corrupt patch\n", __FUNCTION__, __LINE__); + result = 1; + goto Cleanup; + } + /* Read extra string */ + memcpy((*dest_buf) + newpos, patch_buffer + patch_buffer_offset, ctrl[1]); + patch_buffer_offset += ctrl[1]; + }; + *dest_size = newsize; +Cleanup: + //close old file + if (fd >= 0) + close(fd); + if (old) + free(old); + return result; +} diff --git a/bsdiff/ss_bspatch_common.h b/bsdiff/ss_bspatch_common.h new file mode 100644 index 0000000..3075ea2 --- /dev/null +++ b/bsdiff/ss_bspatch_common.h @@ -0,0 +1,17 @@ +#ifndef _SS_BSPATCH_COMMON_H +#define _SS_BSPATCH_COMMON_H 1 + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include + +extern ISzAlloc g_Alloc; + +SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, + UInt64 *unpackSize, unsigned char *dec_data); + +int apply_patch(const char *oldfile, unsigned char *patch_buffer, unsigned char **dest_buf, ssize_t *dest_size); + +#endif /* _SS_BSPATCH_COMMON_H */ -- 2.7.4 From 2648c176f156718c9f83effa65ef82c1367a0a5e Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Wed, 24 Aug 2022 15:58:26 +0200 Subject: [PATCH 16/16] Add dockerfile to provide upgrade-tools in reproductible manner Build with: ./docker-build.sh Run with: docker run --rm -ti --privileged -v /dev:/dev -v /data:/data upgrade-tools:latest Update volume (-v) mappings as needed. Note that --privileged and -v /dev:/dev are needed for loop device mounting to work correctly. /data volume is just example how to transfer tizen images into container and deltas - out of container. Change-Id: I5bd09d446df32eda44dddffb9dcc5bee942cd427 --- Dockerfile | 21 +++++++++++++++++++++ docker-build.sh | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 Dockerfile create mode 100755 docker-build.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ce810e1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:20.04 as build + +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ=UTC + +ADD bsdiff /bsdiff + +RUN apt-get update && apt-get -y --no-install-recommends install libbrotli-dev libdivsufsort-dev git cmake build-essential pkg-config +RUN git clone git://git.tizen.org/platform/upstream/lzma-sdk -b tizen /lzma-sdk +RUN cd /lzma-sdk && cmake -DLIB_INSTALL_DIR=/usr/local/lib . && make install +RUN cd bsdiff && cmake . && make install + +FROM ubuntu:20.04 + +ADD mk_delta /tota-upg/mk_delta/ +ADD scripts /tota-upg/scripts/ +ADD recovery /tota-upg/recovery/ +COPY --from=build /usr/local/bin/ss_bsdiff /usr/local/bin/ss_bspatch /usr/local/bin/ +COPY --from=build /usr/local/lib/liblzma-tool.so.* /usr/local/lib +RUN apt-get update && \ + apt-get install -y --no-install-recommends libbrotli1 libdivsufsort3 python-is-python2 python2 python-apt p7zip-full attr tar file sudo && rm -rf /var/lib/apt/lists/* diff --git a/docker-build.sh b/docker-build.sh new file mode 100755 index 0000000..b6f6486 --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec docker build --no-cache -t upgrade-tools:latest . -- 2.7.4