From: Antoni Adaszkiewicz Date: Mon, 17 Oct 2022 13:31:08 +0000 (+0200) Subject: Add update-info-builder.py, used to generate update-info file. X-Git-Tag: accepted/tizen/unified/20240419.110853~29 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f7413f65c78e92510873ceee5e3af700af51aeeb;p=platform%2Fcore%2Fsystem%2Fupgrade-tools.git Add update-info-builder.py, used to generate update-info file. Use it in delta building scripts to generate file needed to verify it's compatibility with upgraded device. Reduce the ammount of times same partition is mounted/unpacked during delta generation. Change-Id: I6fd4a820cded25588464d2f0279124c7727bc89d --- diff --git a/mk_delta/common/bin/mk_delta.sh b/mk_delta/common/bin/mk_delta.sh index 1aad474..aac874f 100755 --- a/mk_delta/common/bin/mk_delta.sh +++ b/mk_delta/common/bin/mk_delta.sh @@ -89,13 +89,22 @@ fn_set_default_params() DELTA_CFG_PATH=./cfg/delta.cfg UPDATE_CFG_PATH=./cfg/update.cfg UPDATE_INFO_FILE=update-info.ini - UPDATE_INFO_PATH=./data/${UPDATE_INFO_FILE} + UPDATE_INFO_PATH=./${UPDATE_INFO_FILE} SETUP_SCRIPT_PATH=./cfg/setup.sh UPDATE_MANAGER=./data/upgrade-trigger.sh + UPDATE_INFO_BUILDER="update-info-builder.py" + UPDATE_INFO_BUILDER_PATH=${COMMON_BINDIR}/${UPDATE_INFO_BUILDER} LOG_PATH=./data/Delta.log BUILD_STRING_FILE=build_string.txt BUILD_STRING_PATH=./data/${BUILD_STRING_FILE} + + EXTRACTED_FILES_DIR=${STARTING_DIR}"/data" + HAL_TARG_NAME=${EXTRACTED_FILES_DIR}"/hal-model-conf.xml" + ROOTFS_TARG_NAME=${EXTRACTED_FILES_DIR}"/rootfs-model-config.xml" + TIZEN_TARG_NAME=${EXTRACTED_FILES_DIR}"/tizen-build.conf" + BINA_TARG_NAME=${EXTRACTED_FILES_DIR}"/upgrade-support" + BINB_TARG_NAME=${EXTRACTED_FILES_DIR}"/upgrade-apply" # Getting date and minor version MONDATE=$(date +%m%d) i=1 @@ -147,6 +156,19 @@ fn_print_update_cfg() done echo "------------------------------" } +#------------------------------------------------------------------------------ +# Function : +# fn_build_update_info +# Description : +# builds update_info.ini file +# +fn_build_update_info() +{ + echo "PWD=${PWD}" + sudo python3 "${UPDATE_INFO_BUILDER_PATH}" "${HAL_TARG_NAME}" "${ROOTFS_TARG_NAME}" "${TIZEN_TARG_NAME}" ${UPDATE_INFO_PATH} + + rm -f "${HAL_TARG_NAME}" "${ROOTFS_TARG_NAME}" "${TIZEN_TARG_NAME}" +} #------------------------------------------------------------------------------ # Function : @@ -222,54 +244,14 @@ fn_get_tar_file_names() } -#------------------------------------------------------------------------------ -# Function : -# fn_extract_from_image -# -# Description : -# extract some files from given image -# - -fn_extract_from_image() -{ - CUR_DIR=`pwd` - DATA_DIR=./data - IMAGE=$1 - TARGET_FILE=$2 - MNT_PNT=TMP_MNT - mkdir -p ${MNT_PNT} - - fn_get_tar_file_names $IMAGE - tar xvf ${DATA_DIR}/${NEW_TAR_DIR}/${NEW_TAR_FILE} ${IMAGE} - if [ "$?" != "0" ]; then - return 1; - fi - - sudo mount -o loop ${IMAGE} ${MNT_PNT} - if [ "$?" != "0" ]; then - return 1; - fi - - EXTRACT_FILE="${MNT_PNT}${TARGET_FILE}" - if [ -d ${EXTRACT_FILE} ]; then - sudo cp -Rd ${EXTRACT_FILE}/* ${DELTA_DIR} - elif [ -e ${EXTRACT_FILE} ]; then - sudo cp ${EXTRACT_FILE} ${DELTA_DIR} - else - echo "There is no ${TARGET_FILE} in ${IMAGE}" - fi - - sudo umount ${MNT_PNT} - sudo rm -rf ${MNT_PNT} - sudo rm -f ${IMAGE} -} ############################################################################### #================== # Main Start #================== -COMMON_BINDIR=${PWD}/../common/bin +STARTING_DIR=${PWD} +COMMON_BINDIR=${STARTING_DIR}/../common/bin fn_set_default_params @@ -360,11 +342,6 @@ do sudo cp ${SETUP_SCRIPT_PATH} ${DELTA_DIR}/setup.sh fi - #--- extract files which would be appended to delta.tar --- - echo "Extract binaries for update from images" - fn_extract_from_image rootfs.img /usr/libexec/upgrade-support - fn_extract_from_image rootfs.img /usr/bin/upgrade-apply - #--- archive result directory --- cd result/$MONDATE echo "tar result directory" @@ -381,6 +358,9 @@ cd ${DELTA_DIR} sudo cp ${COMMON_BINDIR}/unpack.sh ./ # Ensure essential files are at beginning of archive (metadata, scripts and binaries to perform upgrade) +fn_build_update_info +cp "${BINB_TARG_NAME}" . +cp "${BINA_TARG_NAME}"/* . touch *.txt *.cfg *.ini *.sh *.ua sha1sum * > checksum.SHA1 sudo tar --overwrite -cpf ../delta.tar $(ls -1td *) diff --git a/mk_delta/common/bin/mk_part_delta.sh b/mk_delta/common/bin/mk_part_delta.sh index f71c9e4..d899c88 100755 --- a/mk_delta/common/bin/mk_part_delta.sh +++ b/mk_delta/common/bin/mk_part_delta.sh @@ -7,7 +7,7 @@ #------------------------------------------------------------------------------ # Function : -# fn_mk_attribute +# fn_print_line # fn_print_line() @@ -18,52 +18,91 @@ fn_print_line() echo "*******************************************************************" } -fn_gen_metadata() +#------------------------------------------------------------------------------ +# Function : +# fn_extract_mounting_wrapper +# +# Params : +# +# Description : +# extract neccessary files/info from binary images, +# by mounting them and calling fn_extract_needed_files() +# +fn_extract_mounting_wrapper() { - if [ "$PART_NAME" != "rootfs" ]; then + if [ "$PART_NAME" != "rootfs" ] && [ "$PART_NAME" != "hal" ]; then return fi + local MNT_PNT_OLD="MNT_PNT_OLD" + local MNT_PNT_NEW="MNT_PNT_NEW" - echo "===== Start writing image versions =====" - local TMP_MNT_PNT="TMP_MNT_PNT" - local BUILD_STRING_FILE="build_string.txt" - mkdir -p ${TMP_MNT_PNT} - - tar xvf ${OLD_TAR_DIR}/${OLD_TAR_FILE} ${PART_IMG_ORG} - sudo mount -o loop ${PART_IMG_ORG} ${TMP_MNT_PNT} - local OLD_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Date=" | sed "s/Date=//" | sed "s/;$//"` - sudo umount ${TMP_MNT_PNT} - rm -rf ${PART_IMG_ORG} - - tar xvf ${NEW_TAR_DIR}/${NEW_TAR_FILE} ${PART_IMG_ORG} - sudo mount -o loop ${PART_IMG_ORG} ${TMP_MNT_PNT} - local NEW_BUILD_STRING=`cat ${TMP_MNT_PNT}/etc/info.ini | grep "Date=" | sed "s/Date=//" | sed "s/;$//"` - local BINOUT="$(LC_ALL=C LANG=C file -b ${TMP_MNT_PNT}/bin/ls)" - sudo umount ${TMP_MNT_PNT} - rm -rf ${PART_IMG_ORG} + mkdir -p ${MNT_PNT_OLD} + mkdir -p ${MNT_PNT_NEW} - case "${BINOUT}" in - # ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, ... - *ELF*64-bit*x86-64*) ARCHITECTURE=x86-64;; + sudo mount -o loop,ro "${PART_IMG_OLD}" ${MNT_PNT_OLD} + sudo mount -o loop,ro "${PART_IMG_NEW}" ${MNT_PNT_NEW} - # ELF 32-bit LSB shared object, ARM, EABI5 version, dynamically linked, interpreter /lib/ld-linux.so.3, ... - *ELF*32-bit*ARM,*) ARCHITECTURE=arm32;; + fn_extract_needed_files ${MNT_PNT_OLD} ${MNT_PNT_NEW} - # ELF 64-bit LSB pie executable, ARM aarch64, version 1 (GNU/Linux), dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, ... - *ELF*64-bit*ARM*aarch64*) ARCHITECTURE=arm64;; - esac + sudo umount ${MNT_PNT_OLD} + sudo umount ${MNT_PNT_NEW} - echo "${BUILD_STRING_FILE} : ${OLD_BUILD_STRING}@${NEW_BUILD_STRING}" - echo -n ${OLD_BUILD_STRING}@${NEW_BUILD_STRING} > ${BUILD_STRING_FILE} + rm -rf ${MNT_PNT_OLD} + rm -rf ${MNT_PNT_NEW} +} - sed -e "s,@SOURCE_VERSION@,${OLD_BUILD_STRING},g" \ - -e "s,@TARGET_VERSION@,${NEW_BUILD_STRING},g" \ - -e "s,@ARCHITECTURE@,${ARCHITECTURE},g" \ - -e "s,@IMAGE_TYPE@,all,g" \ - < "../${UPDATE_INFO_PATH}" > "${UPDATE_INFO_FILE}" +#------------------------------------------------------------------------------ +# Function : +# fn_extract_needed_files +# +# Params : +# $1 - mount point of older partiton +# $2 - mount point of newer partition +# +# Description : +# extract neccessary files/info from already mounted partitions +# +fn_extract_needed_files() +{ + # mount/umount as rarely as possible + local MNT_PNT_OLD=$1 + local MNT_PNT_NEW=$2 + + if [ "$PART_NAME" == "hal" ]; then + local HAL_TARG_NAME="hal-model-conf.xml" + local HAL_MODEL_CONFIG_PATH=${MNT_PNT_OLD}"/etc/config/model-config.xml" + + # exctarct files/binaries + cp "${HAL_MODEL_CONFIG_PATH}" ./${HAL_TARG_NAME} + + elif [ "$PART_NAME" == "rootfs" ]; then + # TODO names + export + local ROOTFS_TARG_NAME="rootfs-model-config.xml" + local TIZEN_TARG_NAME="tizen-build.conf" + local BINA_TARG_NAME="upgrade-support" + local BINB_TARG_NAME="upgrade-apply" + + local ROOTFS_MODEL_CONFIG_PATH=${MNT_PNT_OLD}"/etc/config/model-config.xml" + local TIZEN_BUILD_CONFIG_PATH=${MNT_PNT_OLD}"/etc/tizen-build.conf" + local BINARY_A_PATH=${MNT_PNT_OLD}"/usr/libexec/"${BINA_TARG_NAME} + local BINARY_B_PATH=${MNT_PNT_OLD}"/usr/bin/"${BINB_TARG_NAME} + + # build string building + echo "===== Start writing image versions =====" + local BUILD_STRING_FILE="build_string.txt" + local OLD_BUILD_STRING=`cat ${MNT_PNT_OLD}/etc/info.ini | grep "Date=" | sed "s/Date=//" | sed "s/;$//"` + local NEW_BUILD_STRING=`cat ${MNT_PNT_NEW}/etc/info.ini | grep "Date=" | sed "s/Date=//" | sed "s/;$//"` + echo "${BUILD_STRING_FILE} : ${OLD_BUILD_STRING}@${NEW_BUILD_STRING}" + echo -n ${OLD_BUILD_STRING}@${NEW_BUILD_STRING} > ${BUILD_STRING_FILE} + echo "===== Finish writing image versions =====" + + # extract files/binaries + cp "${ROOTFS_MODEL_CONFIG_PATH}" ./${ROOTFS_TARG_NAME} + cp "${TIZEN_BUILD_CONFIG_PATH}" ./${TIZEN_TARG_NAME} + cp -r "${BINARY_A_PATH}" ./${BINA_TARG_NAME} + cp "${BINARY_B_PATH}" ./${BINB_TARG_NAME} - rm -rf ${TMP_MNT_PNT} - echo "===== Finish writing image versions =====" + fi } #------------------------------------------------------------------------------ @@ -151,6 +190,9 @@ fn_mk_full_img() fi sudo mv ${PART_IMG_ORG} ${PART_IMG_NEW} + # after extraction, get neccessary files + fn_extract_mounting_wrapper + if [ ! "z${OLD_TAR_FILE}" = "z" ]; then #---- check whether the binaries are same ---- sudo diff ${PART_IMG_OLD} ${PART_IMG_NEW} @@ -205,6 +247,9 @@ fn_mk_delta_img_core() fi sudo mv ${PART_IMG_ORG} ${PART_IMG_NEW} + # after extraction, get neccessary files + fn_extract_mounting_wrapper + #---- check whether the binaries are same ---- sudo diff ${PART_IMG_OLD} ${PART_IMG_NEW} if [ "$?" = "0" ]; then @@ -257,7 +302,6 @@ fn_mk_delta_img() #CFG_XML=${PART_NAME}_IMG.xml mkdir ${OUTPUT_DIR}/i - fn_gen_metadata fn_mk_delta_img_core if [ "$?" != "0" ]; then return 1 @@ -337,6 +381,9 @@ fn_mk_delta_fs_core() return 1; fi + # after mounting + fn_extract_needed_files "${MNT_PNT_OLD}" "${MNT_PNT_NEW}" + #---- remove unnecessary files & directories ---- for ff in ${EXCLUDE_FILES} do @@ -380,7 +427,7 @@ fn_mk_delta_fs_core() #------------------------------------------------------------------------------ # Function : -# Mk_delta_fs +# fn_mk_delta_fs # # Description : # generate delta file for filesystem type partition @@ -432,7 +479,6 @@ fn_mk_delta_fs() MNT_PNT_OLD=${BASE_OLD}/ MNT_PNT_NEW=${BASE_NEW}/ - fn_gen_metadata fn_mk_delta_fs_core if [ "$?" != "0" ]; then return 1 @@ -534,8 +580,6 @@ CFG_DIR=./cfg #XML_DIR=./xml COMMON_BINDIR=${CUR_DIR}/../common/bin UPDATE_CFG_PATH=./cfg/update.cfg -UPDATE_INFO_FILE=update-info.ini -UPDATE_INFO_PATH=./cfg/${UPDATE_INFO_FILE}.in #--- check if the current working directory is the parent directory of bin, data, cfg, xml --- TEST_DIR=${DATA_DIR} @@ -590,7 +634,7 @@ echo "[new] ${NEW_TAR_FILE} contains ${PART_BIN}" #--- generate delta binary --- -cd ${DATA_DIR} +cd ${DATA_DIR} || exit 1 OUTPUT_DIR=${PART_NAME}_OUT if [ ! -d ${OUTPUT_DIR} ]; then diff --git a/mk_delta/common/bin/update-info-builder.py b/mk_delta/common/bin/update-info-builder.py new file mode 100755 index 0000000..64bc59d --- /dev/null +++ b/mk_delta/common/bin/update-info-builder.py @@ -0,0 +1,179 @@ +#!/usr/bin/python3 + +''' + +Copyright © 2022 Samsung Electronics Co., Ltd.. All rights reserved. + +Permission is hereby granted, without written agreement and without license or royalty fees, to use, copy, modify, and distribute +this software and its documentation for any purpose, provided that the above copyright notice and the following two paragraphs appear +in all copies of this software. + +IN NO EVENT SHALL SAMSUNG ELECTRONICS CO., LTD. BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF SAMSUNG ELECTRONICS CO., LTD. HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +SAMSUNG ELECTRONICS CO., LTD. SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND SAMSUNG ELECTRONICS CO., LTD. HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +''' + +''' +This script is used to generate an info file, which is to be attached to a delta of any type. +The file contains specific details about an image that a particular delta will upgrade. +Sample file (with currently supported image details) +<<< + model_name=rpi4 + manufacturer=Tizen + device_type=IoT_Headless + tz_build_release_name=Tizen7/Unified + tz_build_arch=aarch64 + tz_build_date=20220922_060719 +>>> +''' + + +import xml.etree.ElementTree as ET +import sys +import os +import argparse + + +TIZEN_PREFIX = "tizen.org" +SYSTEM_PREFIX = os.path.join(TIZEN_PREFIX, "system") +BUILD_PREFIX = "TZ_BUILD_" + + +# following three could be simpler, but this way its easier in case we add more variables to check +def generate_hal_model_config_set(): + hal_list = [ + 'model_name' + ] + return set(os.path.join(SYSTEM_PREFIX, elem) for elem in hal_list) + + +def generate_rootfs_model_config_set(): + rootfs_list = [ + 'manufacturer', + 'device_type' + ] + return set(os.path.join(SYSTEM_PREFIX, elem) for elem in rootfs_list) + + +def generate_tizen_build_set(): + tizen_list = [ + 'RELEASE_NAME', + 'ARCH', + 'DATE' + ] + return set(BUILD_PREFIX + elem for elem in tizen_list) + + +def get_dict_from_text_file(path, set_info): + tmp_dict = {} + with open(path, 'r') as reader: + lines = (line.rstrip() for line in reader.readlines()) + lines = (line for line in lines if line) + for line in lines: + var_name, var_value = line.split('=') + var_name.rstrip() + var_value.lstrip() + if var_name in set_info: + set_info.remove(var_name) + # for consistent formatting + tmp_dict[var_name.lower()] = var_value + if len(set_info) == 0: + return tmp_dict + + return None + + +def get_dict_from_xml_file(path, set_info): + tmp_dict = {} + + xml_tree = ET.parse(path) + xml_root = xml_tree.getroot() + + for elem in xml_root.findall('./platform/key'): + name = elem.get('name') + if name in set_info: + set_info.remove(name) + tmp_dict[name] = elem.text + if len(set_info) == 0: + return tmp_dict + + return None + + +def generate_main_dict(args): + main_dict = {} + + hal_set = generate_hal_model_config_set() + rootfs_set = generate_rootfs_model_config_set() + tizen_set = generate_tizen_build_set() + + hal_dict = get_dict_from_xml_file(args.hal_model_config_path, hal_set) + if not hal_dict: + print(f'{args.hal_model_config_path}: error parsing file', file=sys.stderr) + return None + + rootfs_dict = get_dict_from_xml_file(args.rootfs_model_config_path, rootfs_set) + if not rootfs_dict: + print(f'{args.rootfs_model_config_path}: error parsing file', file=sys.stderr) + return None + + tizen_dict = get_dict_from_text_file(args.tizen_build_config_path, tizen_set) + if not tizen_dict: + print(f'{args.tizen_build_config_path}: error parsing file', file=sys.stderr) + return None + + main_dict.update(hal_dict) + main_dict.update(rootfs_dict) + main_dict.update(tizen_dict) + + return main_dict + + +def create_parser(): + parser = argparse.ArgumentParser(description='Generate a formated update-info.ini file used to verify whether \ + delta archive used during update is compatible with our device. This file will be created using three files.') + + parser.add_argument('hal_model_config_path', type=str, help='path to a file originally at path_to_hal/etc/config/model-config.xml') + parser.add_argument('rootfs_model_config_path', type=str, help='path to a file originally at path_to_rootfs/etc/config/model-config.xml') + parser.add_argument('tizen_build_config_path', type=str, help='path to a file originally at path_to_rootfs/etc/tizen-build.conf') + parser.add_argument('output_file', type=str, help='path to an output file') + + return parser + + +def main(): + parser = create_parser() + args = parser.parse_args() + + for index, (arg_name, arg_value) in enumerate(vars(args).items(), start=1): + # we do not want output file to exist + if index == len(vars(args)): + if os.path.exists(arg_value): + parser.error(f'{arg_name} = {arg_value} is a file that already exists!') + break + if not (os.path.exists(arg_value) and os.path.isfile(arg_value)): + parser.error(f'{arg_name} = {arg_value} is not a valid path!') + + print('--- Generate update info file ---') + update_data = generate_main_dict(args) + if not update_data: + # TODO make this exception more verbose + raise Exception + + with open(args.output_file, 'w') as writer: + for key, value in update_data.items(): + _, key = os.path.split(key) + key = key.split()[-1] + value = value.replace('\"', '') + writer.write(f'{key}={value}\n') + + print('--- Generating update info file successful ---') + return + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/mk_delta/rpi4-headed/cfg/delta.cfg b/mk_delta/rpi4-headed/cfg/delta.cfg deleted file mode 120000 index db2bab9..0000000 --- a/mk_delta/rpi4-headed/cfg/delta.cfg +++ /dev/null @@ -1 +0,0 @@ -../../rpi4/cfg/delta.cfg \ No newline at end of file diff --git a/mk_delta/rpi4-headed/cfg/update-info.ini.in b/mk_delta/rpi4-headed/cfg/update-info.ini.in deleted file mode 100644 index 57d5d10..0000000 --- a/mk_delta/rpi4-headed/cfg/update-info.ini.in +++ /dev/null @@ -1,7 +0,0 @@ -deviceType=IoT_Headed -deviceModel=rpi4 -vendorName=Tizen -imageType=@IMAGE_TYPE@ -architecture=@ARCHITECTURE@ -sourceVersion=@SOURCE_VERSION@ -targetVersion=@TARGET_VERSION@ diff --git a/mk_delta/rpi4-headed/data/new_tar/.gitignore b/mk_delta/rpi4-headed/data/new_tar/.gitignore deleted file mode 100755 index e69de29..0000000 diff --git a/mk_delta/rpi4-headed/data/old_tar/.gitignore b/mk_delta/rpi4-headed/data/old_tar/.gitignore deleted file mode 100755 index e69de29..0000000 diff --git a/mk_delta/rpi4-headless/cfg/delta.cfg b/mk_delta/rpi4-headless/cfg/delta.cfg deleted file mode 120000 index db2bab9..0000000 --- a/mk_delta/rpi4-headless/cfg/delta.cfg +++ /dev/null @@ -1 +0,0 @@ -../../rpi4/cfg/delta.cfg \ No newline at end of file diff --git a/mk_delta/rpi4-headless/cfg/update-info.ini.in b/mk_delta/rpi4-headless/cfg/update-info.ini.in deleted file mode 100644 index fbc25a8..0000000 --- a/mk_delta/rpi4-headless/cfg/update-info.ini.in +++ /dev/null @@ -1,7 +0,0 @@ -deviceType=IoT_Headless -deviceModel=rpi4 -vendorName=Tizen -imageType=@IMAGE_TYPE@ -architecture=@ARCHITECTURE@ -sourceVersion=@SOURCE_VERSION@ -targetVersion=@TARGET_VERSION@ diff --git a/mk_delta/rpi4-headless/data/new_tar/.gitignore b/mk_delta/rpi4-headless/data/new_tar/.gitignore deleted file mode 100755 index e69de29..0000000 diff --git a/mk_delta/rpi4-headless/data/old_tar/.gitignore b/mk_delta/rpi4-headless/data/old_tar/.gitignore deleted file mode 100755 index e69de29..0000000