From 007bcdf10bcb8e95eb98f45cd1d94183ba87beda Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Mon, 20 Jun 2022 13:28:31 +0200 Subject: [PATCH 01/16] Save a checksum of all files contained in delta archive Change-Id: I82563afd2ef30784ece716a7b9a090241a159b95 --- mk_delta/common/bin/mk_delta.sh | 1 + mk_delta/common/bin/mk_delta_full.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/mk_delta/common/bin/mk_delta.sh b/mk_delta/common/bin/mk_delta.sh index a3412c5..dc15cce 100755 --- a/mk_delta/common/bin/mk_delta.sh +++ b/mk_delta/common/bin/mk_delta.sh @@ -383,6 +383,7 @@ sudo cp ${COMMON_BINDIR}/unpack.sh ./ # Ensure essential files are at beginning of archive (metadata, scripts and binaries to perform upgrade) touch *.txt *.cfg *.ini *.sh *.ua +sha1sum * > checksum.SHA1 sudo tar --overwrite -cpf ../delta.tar $(ls -1td *) SIGN_PKCS_FILE=$1 diff --git a/mk_delta/common/bin/mk_delta_full.sh b/mk_delta/common/bin/mk_delta_full.sh index 5512f27..dbdfd79 100755 --- a/mk_delta/common/bin/mk_delta_full.sh +++ b/mk_delta/common/bin/mk_delta_full.sh @@ -202,6 +202,7 @@ fn_make_delta_full() #--- archive result directory --- pushd result/$MONDATE/FW_DELTA/DELTA/ + sha1sum * > checksum.SHA1 echo "tar result directory" sudo tar cf ../delta.tar * popd -- 2.7.4 From a439495c6d89e7d62ef1c05fb21f79214f59d77f Mon Sep 17 00:00:00 2001 From: Antoni Date: Fri, 15 Jul 2022 15:15:25 +0200 Subject: [PATCH 02/16] Remove bare excepts in CreatePatch.py Change-Id: I78c7a4d88057f21eaf5fb708a94c0a7378a0b742 --- mk_delta/common/bin/CreatePatch.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index 75c371d..f449e19 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -198,9 +198,9 @@ def main(): logging.info('Time start [%s] - Time end [%s]' % (start, end)) print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end)) - except: + except Exception as exc: logging.error('Usage: {} '.format(os.path.basename(sys.argv[0]))) - raise + raise exc def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH): f = open(UPDATE_CFG_PATH, 'r') @@ -833,17 +833,17 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi if not os.path.isdir(destdir): try: os.makedirs(destdir) - except: + except Exception as exc: logging.critical('Error in NEW files DIR entry -%s' % destdir) - raise + raise exc try: if not stat.S_ISFIFO(os.stat(dst_file).st_mode): shutil.copy2(dst_file, destpath) logging.debug('New files copied from- %s to- %s' % (dst_file, destpath)) - except: + except Exception as exc: logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath)) - raise + raise exc for elt in Dir_Added: newfiles_dest_path = 'run/upgrade-sysroot/' -- 2.7.4 From 16f6b85791d9e6157b2480faa6f75094b2fd20e5 Mon Sep 17 00:00:00 2001 From: Antoni Date: Mon, 18 Jul 2022 13:33:12 +0200 Subject: [PATCH 03/16] Remove unnecessary difflines() function and unnecessary closing of files in CreatePatch.py Change-Id: I6abeeca4ed203d0718ca04b51809220fafe11f56 --- mk_delta/common/bin/CreatePatch.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index f449e19..cc79811 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -301,10 +301,6 @@ def path_head(path): head, tail = ntpath.split(path) return head -def difflines(list1, list2): - c = set(list1).union(set(list2)) - d = set(list1).intersection(set(list2)) - return list(c-d) #Creating Diff between OLD and NEW attribute files v12 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE): @@ -316,16 +312,11 @@ def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE): with open(ATTR_NEW, 'r') as f_new: lines2 = set(f_new.read().splitlines()) - lines = difflines(lines2, lines1) + lines = set.difference(lines2, lines1) with open(ATTR_FILE, 'w+') as file_out: for line in lines: - if line not in lines1: - logging.info('Diff_AttrFiles - %s' % line) - file_out.write(line+'\n') - - f_new.close() - f_old.close() - file_out.close() + logging.info('Diff_AttrFiles - %s' % line) + file_out.write(line+'\n') -- 2.7.4 From 8c54925a68ddb91b2e3fda9489ac318bad5b1ac6 Mon Sep 17 00:00:00 2001 From: Antoni Date: Mon, 18 Jul 2022 13:39:35 +0200 Subject: [PATCH 04/16] Fix inconsistent code formating and indentations in CreatePatch.py Change-Id: I7634bd462dd8b875c816540b3ca0720cde93b2ab --- mk_delta/common/bin/CreatePatch.py | 417 ++++++++++++++++++------------------- 1 file changed, 208 insertions(+), 209 deletions(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index cc79811..037f165 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -46,6 +46,7 @@ Want to extend the same script for entire DIFF generation and replace TOTAlib.sh Catching errors at all stages. SHOULD exit & return error in case of failure ''' + def global_paths(): global DIFF_UTIL global ZIPUTIL @@ -70,6 +71,7 @@ def global_paths(): global VERBATIM_LIST global MEM_FILE + COMMON_BIN_PATH = "../../common/bin/" DIFF_UTIL = "/usr/local/bin/ss_bsdiff" DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch" @@ -96,12 +98,13 @@ MEM_FILE = "NULL" COMPRESSION_LZMA = "lzma" COMPRESSION_BROTLI = "brotli" -SUPPORT_RENAME = "TRUE" #Use appropriate name +SUPPORT_RENAME = "TRUE" # Use appropriate name SUPPORT_CONTAINERS = "FALSE" SUPPORT_VERBATIM = "TRUE" TEST_MODE = "FALSE" + def main(): logging.basicConfig(filename=LOGFILE, level=logging.DEBUG) global AttributeFile @@ -126,27 +129,27 @@ def main(): if len(sys.argv) == 9: ATTR_OLD = sys.argv[6] ATTR_NEW = sys.argv[7] - UPDATE_CFG_PATH = '../'+sys.argv[8] + UPDATE_CFG_PATH = '../' + sys.argv[8] GenerateDiffAttr = "TRUE" elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]: if len(sys.argv) == 7: #Use path in better way - UPDATE_CFG_PATH = '../'+sys.argv[6] + UPDATE_CFG_PATH = '../' + sys.argv[6] global DIFF_UTIL global DIFFPATCH_UTIL if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)): - DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL - DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL + DIFF_UTIL = COMMON_BIN_PATH + DIFF_UTIL + DIFFPATCH_UTIL = COMMON_BIN_PATH + DIFFPATCH_UTIL if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)): print >> sys.stderr, "Diff Util Does NOT exist -- ABORT" - logging.info ('Diff Util Does NOT exist -- ABORT') + logging.info('Diff Util Does NOT exist -- ABORT') sys.exit(1) start = datetime.datetime.now().time() logging.info('*************** ENTERED PYTHON SCRIPT *****************') - logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s] [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW)) + logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s] [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]' % (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW)) ensure_dir_exists(OUT_DIR) if GenerateDiffAttr == "TRUE": @@ -154,11 +157,10 @@ def main(): print >> sys.stderr, "Attributes missing -- ABORT" sys.exit(1) - # Should check if APT is supported on other linux flavours cache = apt.Cache() if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed: - logging.info ('Basic utils installed') + logging.info('Basic utils installed') else: print >> sys.stderr, "Basic utils missing -- ABORT" sys.exit(1) @@ -172,7 +174,7 @@ def main(): SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI) elif UPDATE_TYPE == DELTA_FS: AttributeFile = ATTR_NEW - ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT + ATTR_FILE = OUT_DIR + '/' + PART_NAME + ATTR_DOC_EXT Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE) Old_files, Old_dirs = Get_Files(BASE_OLD) New_files, New_dirs = Get_Files(BASE_NEW) @@ -181,7 +183,6 @@ def main(): if not UPDATE_CFG_PATH == EMPTY: SS_update_cfg(PART_NAME, UPDATE_CFG_PATH) - elif UPDATE_TYPE == EXTRA: print('UPDATE_TYPE ---- EXTRA') else: @@ -202,6 +203,7 @@ def main(): logging.error('Usage: {} '.format(os.path.basename(sys.argv[0]))) raise exc + def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH): f = open(UPDATE_CFG_PATH, 'r') lines = f.readlines() @@ -211,22 +213,23 @@ def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH): ConfigItems = line.split() if ConfigItems[0] == DELTA_BIN: DELTA = ConfigItems[1] - logging.info ('Updating %s config' % DELTA_BIN) + logging.info('Updating %s config' % DELTA_BIN) line = line.rstrip('\n') Value = MEM_REQ - line = line.replace(line, line+'\t'+str(Value)+'\n') + line = line.replace(line, line + '\t' + str(Value) + '\n') f.write(line) else: f.write(line) f.close() + def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD): #for sizes - oldsize_d= os.path.getsize(BASE_OLD) - newsize_d= os.path.getsize(BASE_NEW) - SHA_BIN_DEST= hash_file(BASE_NEW) - SHA_BIN_BASE=hash_file(BASE_OLD) + oldsize_d = os.path.getsize(BASE_OLD) + newsize_d = os.path.getsize(BASE_NEW) + SHA_BIN_DEST = hash_file(BASE_NEW) + SHA_BIN_BASE = hash_file(BASE_OLD) #incase UPDATE CFG is empty DELTA = DELTA_BIN @@ -240,27 +243,26 @@ def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COM for line in lines: ConfigItems = line.split() if ConfigItems[0] == DELTA_BIN: - logging.info ('Updating %s config' % DELTA_BIN) + logging.info('Updating %s config' % DELTA_BIN) DELTA = ConfigItems[1] line = line.rstrip('\n') - line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n') + line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n') f.write(line) else: f.write(line) f.close() patchLoc = '%s/%s' % (OUT_DIR, DELTA) - logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc)) - subprocess.call([DIFF_UTIL,"-c",COMPRESSION_METHOD,BASE_OLD,BASE_NEW,patchLoc]) - + logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW, DELTA_BIN, patchLoc)) + subprocess.call([DIFF_UTIL, "-c", COMPRESSION_METHOD, BASE_OLD, BASE_NEW, patchLoc]) -def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH): - logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN)) - oldsize_d= os.path.getsize(BASE_OLD) - newsize_d= os.path.getsize(BASE_NEW) - SHA_BIN_DEST= hash_file(BASE_NEW) - SHA_BIN_BASE=hash_file(BASE_OLD) +def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH): + logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW, DELTA_BIN)) + oldsize_d = os.path.getsize(BASE_OLD) + newsize_d = os.path.getsize(BASE_NEW) + SHA_BIN_DEST = hash_file(BASE_NEW) + SHA_BIN_BASE = hash_file(BASE_OLD) #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg SS_UpdateSize(BASE_OLD, BASE_NEW) @@ -272,19 +274,21 @@ def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH): for line in lines: ConfigItems = line.split() if ConfigItems[0] == DELTA_BIN: - logging.info ('Updating %s config' % DELTA_BIN) + logging.info('Updating %s config' % DELTA_BIN) DELTA = ConfigItems[1] line = line.rstrip('\n') - line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n') + line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n') f.write(line) else: f.write(line) f.close() + def zipdir(path, zip): - for root, dirs, files in os.walk(path): - for file in files: - zip.write(os.path.join(root, file)) + for root, dirs, files in os.walk(path): + for file in files: + zip.write(os.path.join(root, file)) + def ensure_dir_exists(path): if not os.path.exists(path): @@ -294,15 +298,16 @@ def ensure_dir_exists(path): def path_leaf(path): - head, tail = ntpath.split(path) #This is for windows?? Recheck - return tail + head, tail = ntpath.split(path) # This is for windows?? Recheck + return tail + def path_head(path): - head, tail = ntpath.split(path) - return head + head, tail = ntpath.split(path) + return head -#Creating Diff between OLD and NEW attribute files v12 +# Creating Diff between OLD and NEW attribute files v12 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE): if GenerateDiffAttr == "FALSE": return @@ -316,15 +321,14 @@ def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE): with open(ATTR_FILE, 'w+') as file_out: for line in lines: logging.info('Diff_AttrFiles - %s' % line) - file_out.write(line+'\n') - + file_out.write(line + '\n') def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes): - #Full File Path should MATCH + # Full File Path should MATCH if GenerateDiffAttr == "FALSE": return - FilePath = '"/'+RequestedPath+'"' + FilePath = '"/' + RequestedPath + '"' #print ('FilePath - %s'% (FilePath)) with open(AttributeFile) as f: for line in f: @@ -335,23 +339,24 @@ def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes): File_Attibutes.append(line) -'''This function returns the SHA-1 hash of the file passed into it''' def hash_file(filename): + '''This function returns the SHA-1 hash of the file passed into it''' - # make a hash object - h = hashlib.sha1() + # make a hash object + h = hashlib.sha1() - # open file for reading in binary mode - with open(filename,'rb') as file: - # loop till the end of the file - chunk = 0 - while chunk != b'': - # read only 1024 bytes at a time - chunk = file.read(1024*1024) - h.update(chunk) + # open file for reading in binary mode + with open(filename, 'rb') as file: + # loop till the end of the file + chunk = 0 + while chunk != b'': + # read only 1024 bytes at a time + chunk = file.read(1024 * 1024) + h.update(chunk) + + # return the hex representation of digest + return h.hexdigest() - # return the hex representation of digest - return h.hexdigest() def find_dupes_dir(BASE_OLD, BASE_NEW): dups = {} @@ -395,16 +400,15 @@ def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT): print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW)) for filename in fileListB: - Src_File = BASE_OLD+'/'+filename + Src_File = BASE_OLD + '/' + filename if os.path.islink(Src_File) or os.path.isdir(Src_File): continue # Calculate hash file_hash = hash_file(Src_File) dups[file_hash] = Src_File - for filename in fileListT: - Dest_File = BASE_NEW+'/'+filename + Dest_File = BASE_NEW + '/' + filename if os.path.islink(Dest_File) or os.path.isdir(Dest_File): continue # Calculate hash @@ -419,11 +423,12 @@ def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT): logging.info('Total Duplicate files %d' % (len(fdupes))) return fdupes + def SS_UpdateSize(src_file, dst_file): global MEM_REQ global MEM_FILE - oldsize_d= os.path.getsize(src_file) - newsize_d= os.path.getsize(dst_file) + oldsize_d = os.path.getsize(src_file) + newsize_d = os.path.getsize(dst_file) if oldsize_d >= newsize_d: Max = newsize_d else: @@ -433,7 +438,6 @@ def SS_UpdateSize(src_file, dst_file): MEM_FILE = dst_file - def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE): print('Going from %d files to %d files' % (len(Old_files), len(New_files))) logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files))) @@ -453,8 +457,6 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi files_New_List = {} MyDict_Patches = {} - - PWD = os.getcwd() # Generate NEW List @@ -470,13 +472,12 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi files_removed.append(elt) logging.info('Old files %s' % elt) - for elt in Old_dirs: #print('List of Old Dirs %s' % elt) # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done. if elt not in New_dirs: Dir_removed.append(elt) - logging.info('Old Dirs %s' % elt+'/') + logging.info('Old Dirs %s' % elt + '/') for elt in New_dirs: if elt not in Old_dirs: @@ -486,9 +487,9 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi # What files have changed contents but not name/path? for elt in New_files: if elt in Old_files: - #Both are symbolic linkes and they differ - src_file = BASE_OLD+'/'+elt - dst_file = BASE_NEW+'/'+elt + # Both are symbolic linkes and they differ + src_file = BASE_OLD + '/' + elt + dst_file = BASE_NEW + '/' + elt #print('Files Changed - %s -%s' % (src_file,dst_file)) if os.path.islink(src_file) and os.path.islink(dst_file): if not os.readlink(src_file) == os.readlink(dst_file): @@ -497,7 +498,7 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi logging.info('Sym links Changed - %s' % elt) else: files_unchanged.append(elt) - #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink) + # Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink) elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file): if not filecmp.cmp(src_file, dst_file): files_changed.append(elt) @@ -505,18 +506,17 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi #print('Files Changed - %s' % elt) else: files_unchanged.append(elt) - #File types differ between BASE and TARGET + # File types differ between BASE and TARGET else: logging.info('Files are of diff types but same names Src- %s Des- %s' % (src_file, dst_file)) - #Both file types have changed and they differ - #Case 1: First Delete the OLD entry file type (Be it anything) - #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here + # Both file types have changed and they differ + # Case 1: First Delete the OLD entry file type (Be it anything) + # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here files_removed.append(elt) files_new.append(elt) - # HANDLING VERBATIM - Remove from changed list and delete the entries on device first - #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?) + # This script is called partition wise, So, how do u want to handle it? (specialy for delete case?) print("Check for any verbatim under - %s" % VERBATIM_LIST) if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST): @@ -529,22 +529,22 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi if line in files_new: files_new.remove(line) - #Currently if Version or number is the first character of the file, then we are NOT making any diffs. + # Currently if Version or number is the first character of the file, then we are NOT making any diffs. if SUPPORT_RENAME == "TRUE": for elt in files_removed: - if os.path.isfile(BASE_OLD+'/'+elt): + if os.path.isfile(BASE_OLD + '/' + elt): FileName = path_leaf(elt) - entries = re.split('[0-9]' , FileName) - #Gives the STRING part of NAME. if name starts with version then later part wil b string + entries = re.split('[0-9]', FileName) + # Gives the STRING part of NAME. if name starts with version then later part wil b string #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt)) - #If version is starting at the begining of the string?? shd we hav additional check for such cases?? + # If version is starting at the begining of the string?? shd we hav additional check for such cases?? if len(entries[0]) > 0: files_Del_List.update({entries[0]: elt}) for elt in files_new: - if os.path.isfile(BASE_NEW+'/'+elt): + if os.path.isfile(BASE_NEW + '/' + elt): FileName = path_leaf(elt) - entries = re.split('[0-9]' , FileName) + entries = re.split('[0-9]', FileName) #print('Entires under NEWfiles list after split - %s %s - %s' % (FileName, entries[0], elt)) if len(entries[0]) > 0: files_New_List.update({entries[0]: elt}) @@ -553,8 +553,8 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi #print('Key value pair -%s -%s' % (key, value)) if key in files_New_List: # this file is the same name in both! - src_file = BASE_OLD+'/'+value - dst_file = BASE_NEW+'/'+files_New_List[key] + src_file = BASE_OLD + '/' + value + dst_file = BASE_NEW + '/' + files_New_List[key] olddirpath = path_head(files_New_List[key]) newdirpath = path_head(value) if os.path.islink(src_file) or os.path.islink(dst_file): @@ -573,39 +573,39 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS. ''' Sym_Diff_Cnt = 0 - Sym_New_Cnt = 0; + Sym_New_Cnt = 0 Del_Cnt = 0 New_Cnt = 0 Diff_Cnt = 0 Move_Cnt = 0 Verbatim_Cnt = 0 - SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME - Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w') - Partition_Doc_SymLinks = open(SymLinkDoc,'w') + SymLinkDoc = OUT_DIR + '/' + PART_NAME + SYMLINK_DOC_NAME + Partition_Doc = open(OUT_DIR + '/' + PART_NAME + '.txt', 'w') + Partition_Doc_SymLinks = open(SymLinkDoc, 'w') print("writing diff'ed changed files...") for elt in files_changed: - dst_file = BASE_NEW+'/'+elt - src_file = BASE_OLD+'/'+elt - #Both files are symbolic links and they differ + dst_file = BASE_NEW + '/' + elt + src_file = BASE_OLD + '/' + elt + # Both files are symbolic links and they differ if os.path.islink(dst_file) and os.path.islink(src_file): - #Both are symlinks and they differ + # Both are symlinks and they differ logging.debug(' File Changed is Link %s ' % dst_file) patch = os.readlink(dst_file) Sym_Diff_Cnt = Sym_Diff_Cnt + 1 Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch)) Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes) - #Both are NORMAL files and they differ + # Both are NORMAL files and they differ elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file): - #Both are files and they differ + # Both are files and they differ Diff_Cnt = Diff_Cnt + 1 - patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt)) + patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt)) patchLoc = '%s/%s' % (OUT_DIR, patchName) logging.debug(' File Differ %s %s' % (src_file, dst_file)) SS_UpdateSize(src_file, dst_file) FORMAT = "REG" - ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc]) + ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc]) if ret is not 0: logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file)) files_new.append(elt) @@ -614,9 +614,9 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName)) Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes) - #Both differ but they are of diff types + # Both differ but they are of diff types else: - #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here + # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here files_removed.append(elt) files_new.append(elt) @@ -625,25 +625,25 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi logging.info('Dupes %s -> %s' % (oldpath, newpath)) for elt in files_removed: - src_file = BASE_OLD+'/'+elt - #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE) + src_file = BASE_OLD + '/' + elt + # If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE) if src_file in fdupes.keys(): - dst_file = BASE_NEW+'/'+ fdupes[src_file] + dst_file = BASE_NEW + '/' + fdupes[src_file] logging.debug(' File Moved %s ==> %s' % (src_file, dst_file)) Move_Cnt = Move_Cnt + 1 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file))) files_removed.remove(elt) files_new.remove(fdupes[src_file]) - #Should be placed after removing duplicates, else they will be filtered here. + # Should be placed after removing duplicates, else they will be filtered here. # loop shd b for all NEW files, rather than for all delete files (Current understanding) - # First Step: Sort & Filter out unwanted files + # First Step: Sort & Filter out unwanted files # Minimum condition used is, # 1. File name should match 70% # 2. Extensions should be same # 3. File name length shd b greater than 3 char # 4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break. - # 5. Should consider editdistance for RENAME LOGIC ==> TBD + # 5. Should consider editdistance for RENAME LOGIC ==> TBD Base_DelList = files_removed[:] Base_NewList = files_new[:] @@ -653,11 +653,11 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Filter1 = [] Filter2 = [] - #Remove unwanted items which we cant make diff with for rename logic + # Remove unwanted items which we cant make diff with for rename logic for file in DelList: - if os.path.islink(BASE_OLD+'/'+file): + if os.path.islink(BASE_OLD + '/' + file): continue - elif os.path.isdir(BASE_OLD+'/'+file): + elif os.path.isdir(BASE_OLD + '/' + file): continue else: Filter1.append(file) @@ -666,12 +666,12 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi DelList = Filter1 for file in NewList: - if os.path.islink(BASE_NEW+'/'+file): + if os.path.islink(BASE_NEW + '/' + file): continue - elif os.path.isdir(BASE_NEW+'/'+file): + elif os.path.isdir(BASE_NEW + '/' + file): continue elif len(path_leaf(file)) <= 3: - logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file)) + logging.debug('Ignored for best picks -%s ' % (BASE_NEW + '/' + file)) continue else: Filter2.append(file) @@ -681,34 +681,34 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList))) for new_file in NewList: - R_Flag = 'FALSE'; + R_Flag = 'FALSE' DirPathNew = path_head(new_file) FileNameNew = path_leaf(new_file) DiffSize = 0 - winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file) + winning_patch_sz = os.path.getsize(BASE_NEW + '/' + new_file) New_fs = winning_patch_sz winning_file = '' for del_file in DelList: FileNameOld = path_leaf(del_file) - if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])): + if (FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])): #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file) - #Percentage difference between two file sizes is within 30%, then we consider for diff generation - Del_fs = os.path.getsize(BASE_OLD+'/'+del_file) - v1 = abs(New_fs-Del_fs) - v2 = (New_fs+Del_fs)/2 - if( v2<=0 or ((v1/v2) * 100) > 30 ): + # Percentage difference between two file sizes is within 30%, then we consider for diff generation + Del_fs = os.path.getsize(BASE_OLD + '/' + del_file) + v1 = abs(New_fs - Del_fs) + v2 = (New_fs + Del_fs) / 2 + if(v2 <= 0 or ((v1 / v2) * 100) > 30): logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs)) continue logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs)) - R_Flag = 'TRUE'; - DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file) + R_Flag = 'TRUE' + DiffSize = measure_two_filediffs(BASE_OLD + '/' + del_file, BASE_NEW + '/' + new_file) if (DiffSize < 0.8 * winning_patch_sz): winning_patch_sz = DiffSize winning_file = del_file - elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'): + elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and R_Flag == 'TRUE'): logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file)) - break; + break if len(winning_file) > 0: logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize)) files_renamed.append([new_file, winning_file]) @@ -720,10 +720,10 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi if SUPPORT_RENAME == "TRUE": for elt in files_renamed: - src_file = BASE_OLD+'/'+elt[1] - dst_file = BASE_NEW+'/'+elt[0] + src_file = BASE_OLD + '/' + elt[1] + dst_file = BASE_NEW + '/' + elt[0] Diff_Cnt = Diff_Cnt + 1 - patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1])) + patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1])) #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0])) patchLoc = '%s/%s' % (OUT_DIR, patchName) logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file)) @@ -731,18 +731,18 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file? # What if folder path has numerics?? - if os.path.isdir(src_file) or os.path.isdir(dst_file): - #This case never occurs?? + if os.path.isdir(src_file) or os.path.isdir(dst_file): + # This case never occurs?? Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1])) Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes) - else: #Make sure these files are PROPER and they shd NOT be symlinks + else: # Make sure these files are PROPER and they shd NOT be symlinks if filecmp.cmp(src_file, dst_file): Move_Cnt = Move_Cnt + 1 Diff_Cnt = Diff_Cnt - 1 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file))) else: FORMAT = "REG" - ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc]) + ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc]) if ret is not 0: logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file)) files_new.append(elt) @@ -753,29 +753,26 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi SS_UpdateSize(src_file, dst_file) Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes) - - #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality. - #So that, the rename functionality will NOT create PATCH instead of verbatims. + # HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality. + # So that, the rename functionality will NOT create PATCH instead of verbatims. if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST): with open(VERBATIM_LIST, 'r') as F_News: lines = set(F_News.read().splitlines()) for line in lines: - if not line in files_new: - if os.path.exists(BASE_NEW+'/'+line): + if line not in files_new: + if os.path.exists(BASE_NEW + '/' + line): files_new.append(line) - Verbatim_Cnt = Verbatim_Cnt+1 - logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line) - - + Verbatim_Cnt = Verbatim_Cnt + 1 + logging.debug("Added to list of verbatims -%s" % BASE_NEW + '/' + line) for elt in files_removed: - #if files are part of patches after renaming, we shd remove them as part of removed. - src_file = BASE_OLD+'/'+elt + # if files are part of patches after renaming, we shd remove them as part of removed. + src_file = BASE_OLD + '/' + elt if os.path.islink(src_file): Partition_Doc.write('DEL:SYM:%s\n' % (elt)) elif os.path.isdir(src_file): - #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed. + # If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed. Partition_Doc.write('DEL:REG:%s:NA\n' % (elt)) else: Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file))) @@ -784,32 +781,31 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Dir_removed.sort(reverse=True) for elt in Dir_removed: - #if Dir is empty, add it to the removed list. - src_file = BASE_OLD+'/'+elt - #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end. - #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary) + # if Dir is empty, add it to the removed list. + src_file = BASE_OLD + '/' + elt + # Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end. + # It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary) if os.path.isdir(src_file): Partition_Doc.write('DEL:END:%s\n' % (elt)) Del_Cnt = Del_Cnt + 1 logging.debug(' Dir Deleted- %s' % src_file) - for elt in files_new: - dst_file = BASE_NEW+'/'+elt + dst_file = BASE_NEW + '/' + elt newfiles_dest_path = 'run/upgrade-sysroot/' ensure_dir_exists(newfiles_dest_path) if os.path.islink(dst_file): patch = os.readlink(dst_file) logging.debug(' File New Links %s' % elt) Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch)) - #What if this is only a new sym link and folder already exists??? Should recheck + # What if this is only a new sym link and folder already exists??? Should recheck destpath = newfiles_dest_path + elt if not os.path.exists(path_head(destpath)): os.makedirs(path_head(destpath)) logging.info('New SymLink - Adding missing Dir') #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes) Sym_New_Cnt = Sym_New_Cnt + 1 - elif os.path.isdir(dst_file): # We create just empty directory here + elif os.path.isdir(dst_file): # We create just empty directory here destpath = newfiles_dest_path + elt if not os.path.exists(destpath): os.makedirs(destpath) @@ -845,16 +841,16 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi logging.debug(' DirList New Dir %s' % destpath) New_Cnt = New_Cnt + 1 - #Base directory should be system + # Base directory should be system print 'Compressing New files' if (New_Cnt > 0 or Sym_New_Cnt > 0): WorkingDir = os.getcwd() - os.chdir(os.getcwd()+"/"+NEW_FILES_PATH) + os.chdir(os.getcwd() + "/" + NEW_FILES_PATH) logging.info('Curr Working Dir - %s' % os.getcwd()) - os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE) - shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR) - #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression - SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME) + os.system(ZIPUTIL + NEW_FILES_PATH + " >> " + LOGFILE) + shutil.move(NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR) + # New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression + SS_UpdateSize(WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME) os.chdir(WorkingDir) shutil.rmtree(NEW_FILES_PATH) # use 7z a system.7z ./* @@ -868,8 +864,8 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt)) print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt)) - #There could be duplicates, TODO, can check before adding.. - ATTR_FILE_D = open(ATTR_FILE,'a+') + # There could be duplicates, TODO, can check before adding.. + ATTR_FILE_D = open(ATTR_FILE, 'a+') for elt in File_Attibutes: ATTR_FILE_D.write(elt) for elt in Sym_Attibutes: @@ -878,32 +874,32 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi ATTR_FILE_D.close() Partition_Doc_SymLinks.close() - Partition_Read_SymLinks = open(SymLinkDoc,'r+') + Partition_Read_SymLinks = open(SymLinkDoc, 'r+') Partition_Doc.write(Partition_Read_SymLinks.read()) Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt)) Partition_Doc_SymLinks.close() Partition_Doc.close() os.remove(SymLinkDoc) - if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0: - print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR)) + if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0: + print('No Delta Generated for %s - %s' % (PART_NAME, OUT_DIR)) logging.info('No Delta Generated for %s' % PART_NAME) shutil.rmtree(OUT_DIR) def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir): - #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED. + # CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED. print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir - shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp') - temp_apk = '../'+g_output_dir+'/'+b_apk - Patch = 'Patch_'+b_apk + shutil.copy2(g_output_dir + '/' + b_apk, g_output_dir + '/temp') + temp_apk = '../' + g_output_dir + '/' + b_apk + Patch = 'Patch_' + b_apk ensure_dir_exists(Patch) - shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk) + shutil.copy2(g_output_dir + '/' + b_apk, Patch + '/' + b_apk) - #Size issue on Device side?? shd check this - subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch]) - with open(g_output_dir+'/PATCH.txt', 'r') as f_new: + # Size issue on Device side?? shd check this + subprocess.call(['unzip', '-q', Patch + '/' + b_apk, '-d', Patch]) + with open(g_output_dir + '/PATCH.txt', 'r') as f_new: lines = set(f_new.read().splitlines()) for line in lines: #print('Action ==> %s' % line) @@ -911,80 +907,84 @@ def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir): Items = line.split('|') Action = Items[0] Path = Items[1] - ActualPath = a_folder+'/'+Path - PatchPath = Patch+'/'+Path - SrcPath = g_output_dir+'/'+path_leaf(Path) + ActualPath = a_folder + '/' + Path + PatchPath = Patch + '/' + Path + SrcPath = g_output_dir + '/' + path_leaf(Path) #print('Action ==> %s Path ==> %s ' % (Action, Path)) if line[0] == 'c': - patchName = g_output_dir+'/'+Items[2] + patchName = g_output_dir + '/' + Items[2] #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName)) - subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName]) + subprocess.call([DIFFPATCH_UTIL, ActualPath, ActualPath, patchName]) WorkingDir = os.getcwd() - os.chdir(WorkingDir+"/"+"temp_a") - subprocess.call(['cp', '--parents', Path, '../'+Patch]) + os.chdir(WorkingDir + "/" + "temp_a") + subprocess.call(['cp', '--parents', Path, '../' + Patch]) os.chdir(WorkingDir) elif line[0] == 's': WorkingDir = os.getcwd() - os.chdir(WorkingDir+"/"+"temp_a") - subprocess.call(['cp', '--parents', Path, '../'+Patch]) + os.chdir(WorkingDir + "/" + "temp_a") + subprocess.call(['cp', '--parents', Path, '../' + Patch]) os.chdir(WorkingDir) else: print('Apply_Container_Delta - Unknown Error') #print('Touch all files and set common attributes for DIFF generation') WorkingDir = os.getcwd() - os.chdir(WorkingDir+"/"+Patch) + os.chdir(WorkingDir + "/" + Patch) CONTAINER_DATE = '200011111111.11' CONTAINER_MODE = '0755' subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';']) subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';']) - subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch]) + subprocess.call(['chmod', '-R', CONTAINER_MODE, '../' + Patch]) print 'Update Intermediate Archive' #subprocess.call(['zip','-ryX', b_apk, '*']) - subprocess.call(['zip','-ryX', b_apk] + glob.glob('*')) + subprocess.call(['zip', '-ryX', b_apk] + glob.glob('*')) os.chdir(WorkingDir) #print('Apply Path completed - Now create diff for this and place in patch folder') #print os.getcwd() - print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk)) - patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk) + print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir + '/' + b_apk, new_apk)) + patchName = ('New' + '_%s' + DIFF_SUFFIX) % (b_apk) patchLoc = '%s/%s' % (g_output_dir, patchName) - subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc]) + subprocess.call([DIFF_UTIL, Patch + '/' + b_apk, new_apk, patchLoc]) - #Only on HOST... for testing + # Only on HOST... for testing if TEST_MODE == 'TRUE': UpgradedName = '%s_Upgraded' % (b_apk) - subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc]) + subprocess.call([DIFFPATCH_UTIL, Patch + '/' + b_apk, UpgradedName, patchLoc]) - #This is file only with NEWS and empty diffs and same files. + # This is file only with NEWS and empty diffs and same files. if TEST_MODE == 'FALSE': - os.remove(g_output_dir+'/'+b_apk) - os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk) + os.remove(g_output_dir + '/' + b_apk) + os.rename(g_output_dir + '/temp', g_output_dir + '/' + b_apk) shutil.rmtree(Patch) + def IsSymlink(info): - return (info.external_attr >> 16) == 0120777 + return (info.external_attr >> 16) == 0120777 + def NewFiles(src, dest): - print src,dest - subprocess.call(['cp','-rp', src,dest]) - #try: + print src, dest + subprocess.call(['cp', '-rp', src, dest]) + #try: #shutil.copytree(src, dest) - #except OSError as e: - # If the error was caused because the source wasn't a directory - #if e.errno == errno.ENOTDIR: - #shutil.copy2(src, dest) - #else: - #print('Directory not copied. Error: %s' % e) + #except OSError as e: + # If the error was caused because the source wasn't a directory + #if e.errno == errno.ENOTDIR: + #shutil.copy2(src, dest) + #else: + #print('Directory not copied. Error: %s' % e) + def measure_two_filediffs(src, dst): patchLoc = 'temp.patch' - subprocess.call([DIFF_UTIL,src,dst,patchLoc]) + subprocess.call([DIFF_UTIL, src, dst, patchLoc]) result_size = os.path.getsize(patchLoc) os.remove(patchLoc) return result_size + def Get_Files(path): all_files = [] all_dirs = [] @@ -992,18 +992,18 @@ def Get_Files(path): for root, directories, filenames in os.walk(path, topdown=False, followlinks=False): for directory in directories: #DirName = os.path.join(root+'/',directory) - DirName = os.path.join(root,directory) + DirName = os.path.join(root, directory) if os.path.islink(DirName): logging.debug('This is symlink pointing to dir -%s' % DirName) all_files.append(os.path.relpath(DirName, path)) elif not os.listdir(DirName): #print('*****Empty Directory******* -%s', DirName) - #This should NOT be appended ??? Empty dir shd b considered + # This should NOT be appended ??? Empty dir shd b considered all_dirs.append(os.path.relpath(DirName, path)) else: all_dirs.append(os.path.relpath(DirName, path)) for filename in filenames: - FileName = os.path.join(root,filename) + FileName = os.path.join(root, filename) all_files.append(os.path.relpath(FileName, path)) all_files.sort() @@ -1012,17 +1012,16 @@ def Get_Files(path): USAGE_DOCSTRING = """ - Generate Delta using BASEOLD AND BASE NEW - Attributes is optional - Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER + Generate Delta using BASEOLD AND BASE NEW + Attributes is optional + Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER """ -def Usage(docstring): - print docstring.rstrip("\n") - print COMMON_DOCSTRING +def Usage(docstring): + print docstring.rstrip("\n") + print COMMON_DOCSTRING if __name__ == '__main__': main() - -- 2.7.4 From a8e1de40958fe37a8dcf62d5d47b9da2c3443bdf Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Thu, 21 Jul 2022 14:48:20 +0200 Subject: [PATCH 05/16] Drop DELTA_FS image generation This commit changes all occurences of DELTA_FS with DELTA_IMAGE. DELTA_FS is considered legacy and should not be used unless necessary. Change-Id: I17dccac5ec532af2db0bb2e385c9871763956b38 --- mk_delta/artik530_710/cfg/delta.cfg | 2 +- mk_delta/rpi3/cfg/delta.cfg | 6 +++--- mk_delta/rpi4/cfg/delta.cfg | 6 +++--- mk_delta/tw1/cfg/delta.cfg | 10 +++++----- mk_delta/tw2/cfg/delta.cfg | 10 +++++----- mk_delta/tw3/cfg/delta.cfg | 10 +++++----- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/mk_delta/artik530_710/cfg/delta.cfg b/mk_delta/artik530_710/cfg/delta.cfg index 2eee482..55dd5b6 100644 --- a/mk_delta/artik530_710/cfg/delta.cfg +++ b/mk_delta/artik530_710/cfg/delta.cfg @@ -1,6 +1,6 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p3 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p3 0 ramdisk ramdisk.img ramdisk.img FULL_IMAGE:BEFORE_BOOT_FOTA /tmp/boot/ramdisk.img 0 ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /tmp/boot/ramdisk-recovery.img 0 diff --git a/mk_delta/rpi3/cfg/delta.cfg b/mk_delta/rpi3/cfg/delta.cfg index 1228590..cab40a7 100644 --- a/mk_delta/rpi3/cfg/delta.cfg +++ b/mk_delta/rpi3/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset BOOT boot.img boot.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p1 0 -hal hal.img hal.img/ DELTA_FS /dev/mmcblk0p10 0 +hal hal.img hal.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p10 0 modules modules.img modules.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p6 0 -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p2 0 -ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p2 0 +ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p8 0 diff --git a/mk_delta/rpi4/cfg/delta.cfg b/mk_delta/rpi4/cfg/delta.cfg index 1228590..09380e1 100644 --- a/mk_delta/rpi4/cfg/delta.cfg +++ b/mk_delta/rpi4/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Filesystem label, bin name (in tar), delta name, update type, blk dev, blk offset BOOT boot.img boot.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p1 0 -hal hal.img hal.img/ DELTA_FS /dev/mmcblk0p10 0 +hal hal.img hal.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p10 0 modules modules.img modules.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p6 0 -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p2 0 -ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p2 0 +ramdisk ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p7 0 ramdisk-recovery ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p8 0 diff --git a/mk_delta/tw1/cfg/delta.cfg b/mk_delta/tw1/cfg/delta.cfg index 8eb9ac6..e8b6ad5 100755 --- a/mk_delta/tw1/cfg/delta.cfg +++ b/mk_delta/tw1/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p13 0 -boot zImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p6 0 -recovery zImage-recovery zImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p13 0 +boot zImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p6 0 +recovery zImage-recovery zImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 diff --git a/mk_delta/tw2/cfg/delta.cfg b/mk_delta/tw2/cfg/delta.cfg index a69bb53..00849c8 100755 --- a/mk_delta/tw2/cfg/delta.cfg +++ b/mk_delta/tw2/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p15 0 -boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 -recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p9 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p5 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p15 0 +boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p9 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p5 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p4 0 diff --git a/mk_delta/tw3/cfg/delta.cfg b/mk_delta/tw3/cfg/delta.cfg index 34ea8dc..5521f94 100755 --- a/mk_delta/tw3/cfg/delta.cfg +++ b/mk_delta/tw3/cfg/delta.cfg @@ -1,8 +1,8 @@ # Configuration for generation of delta # Partition Name (in PIT), bin name (in tar), delta name, update type, blk dev, blk offset -rootfs rootfs.img rootfs.img/ DELTA_FS /dev/mmcblk0p18 0 -boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p11 0 -recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p12 0 -ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 -ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 +rootfs rootfs.img rootfs.img DELTA_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p18 0 +boot dzImage delta.boot FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p11 0 +recovery dzImage-recovery dzImage-recovery FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p12 0 +ramdisk1 ramdisk.img ramdisk.img FULL_IMAGE:AT_BOOT_FOTA /dev/mmcblk0p8 0 +ramdisk2 ramdisk-recovery.img ramdisk-recovery.img FULL_IMAGE:BEFORE_BOOT_FOTA /dev/mmcblk0p7 0 -- 2.7.4 From c82119c8362a4c29933062cd21138662af494016 Mon Sep 17 00:00:00 2001 From: Antoni Date: Mon, 18 Jul 2022 13:42:35 +0200 Subject: [PATCH 06/16] Fix ensure_dir_exists() function behaviour and exception handling Change-Id: I98b5a184714d34784068e840a1bb2636458a752f --- mk_delta/common/bin/CreatePatch.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index 037f165..566b414 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -151,7 +151,11 @@ def main(): logging.info('*************** ENTERED PYTHON SCRIPT *****************') logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s] [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]' % (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW)) - ensure_dir_exists(OUT_DIR) + try: + ensure_dir_exists(OUT_DIR) + except FileExistsError as exc: + logging.error('Argument passed as OUT_DIR - %s is already an existing file' % OUT_DIR) + raise exc if GenerateDiffAttr == "TRUE": if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)): print >> sys.stderr, "Attributes missing -- ABORT" @@ -293,6 +297,8 @@ def zipdir(path, zip): def ensure_dir_exists(path): if not os.path.exists(path): os.makedirs(path) + elif os.path.isfile(path): + raise FileExistsError #shutil.rmtree(path) #os.makedirs(path) @@ -793,7 +799,11 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi for elt in files_new: dst_file = BASE_NEW + '/' + elt newfiles_dest_path = 'run/upgrade-sysroot/' - ensure_dir_exists(newfiles_dest_path) + try: + ensure_dir_exists(newfiles_dest_path) + except FileExistsError as exc: + logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path) + raise exc if os.path.islink(dst_file): patch = os.readlink(dst_file) logging.debug(' File New Links %s' % elt) @@ -834,7 +844,11 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi for elt in Dir_Added: newfiles_dest_path = 'run/upgrade-sysroot/' - ensure_dir_exists(newfiles_dest_path) + try: + ensure_dir_exists(newfiles_dest_path) + except FileExistsError as exc: + logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path) + raise exc destpath = newfiles_dest_path + elt if not os.path.exists(destpath): os.makedirs(destpath) -- 2.7.4 From c893ea37065f54831593ef7a3d1cd6de9ce92f29 Mon Sep 17 00:00:00 2001 From: Antoni Date: Mon, 18 Jul 2022 17:08:46 +0200 Subject: [PATCH 07/16] Remove unused Apply_Container_Delta() function from CreatePatch.py Change-Id: Id8ad395399ba6b631cf1ad2ecb3ce8b201b95e43 --- mk_delta/common/bin/CreatePatch.py | 73 -------------------------------------- 1 file changed, 73 deletions(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index 566b414..43e7f14 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -901,79 +901,6 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi shutil.rmtree(OUT_DIR) -def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir): - - # CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED. - print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir - shutil.copy2(g_output_dir + '/' + b_apk, g_output_dir + '/temp') - temp_apk = '../' + g_output_dir + '/' + b_apk - Patch = 'Patch_' + b_apk - ensure_dir_exists(Patch) - shutil.copy2(g_output_dir + '/' + b_apk, Patch + '/' + b_apk) - - # Size issue on Device side?? shd check this - subprocess.call(['unzip', '-q', Patch + '/' + b_apk, '-d', Patch]) - with open(g_output_dir + '/PATCH.txt', 'r') as f_new: - lines = set(f_new.read().splitlines()) - for line in lines: - #print('Action ==> %s' % line) - #Action, Path, Patch = line.split('|') - Items = line.split('|') - Action = Items[0] - Path = Items[1] - ActualPath = a_folder + '/' + Path - PatchPath = Patch + '/' + Path - SrcPath = g_output_dir + '/' + path_leaf(Path) - #print('Action ==> %s Path ==> %s ' % (Action, Path)) - if line[0] == 'c': - patchName = g_output_dir + '/' + Items[2] - #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName)) - subprocess.call([DIFFPATCH_UTIL, ActualPath, ActualPath, patchName]) - WorkingDir = os.getcwd() - os.chdir(WorkingDir + "/" + "temp_a") - subprocess.call(['cp', '--parents', Path, '../' + Patch]) - os.chdir(WorkingDir) - elif line[0] == 's': - WorkingDir = os.getcwd() - os.chdir(WorkingDir + "/" + "temp_a") - subprocess.call(['cp', '--parents', Path, '../' + Patch]) - os.chdir(WorkingDir) - else: - print('Apply_Container_Delta - Unknown Error') - #print('Touch all files and set common attributes for DIFF generation') - WorkingDir = os.getcwd() - os.chdir(WorkingDir + "/" + Patch) - - CONTAINER_DATE = '200011111111.11' - CONTAINER_MODE = '0755' - subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';']) - subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';']) - subprocess.call(['chmod', '-R', CONTAINER_MODE, '../' + Patch]) - - print 'Update Intermediate Archive' - #subprocess.call(['zip','-ryX', b_apk, '*']) - subprocess.call(['zip', '-ryX', b_apk] + glob.glob('*')) - os.chdir(WorkingDir) - #print('Apply Path completed - Now create diff for this and place in patch folder') - #print os.getcwd() - print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir + '/' + b_apk, new_apk)) - patchName = ('New' + '_%s' + DIFF_SUFFIX) % (b_apk) - patchLoc = '%s/%s' % (g_output_dir, patchName) - - subprocess.call([DIFF_UTIL, Patch + '/' + b_apk, new_apk, patchLoc]) - - # Only on HOST... for testing - if TEST_MODE == 'TRUE': - UpgradedName = '%s_Upgraded' % (b_apk) - subprocess.call([DIFFPATCH_UTIL, Patch + '/' + b_apk, UpgradedName, patchLoc]) - - # This is file only with NEWS and empty diffs and same files. - if TEST_MODE == 'FALSE': - os.remove(g_output_dir + '/' + b_apk) - os.rename(g_output_dir + '/temp', g_output_dir + '/' + b_apk) - shutil.rmtree(Patch) - - def IsSymlink(info): return (info.external_attr >> 16) == 0120777 -- 2.7.4 From f4bef151ec4c81db9d3ab79ed4c54cb6fa3d9f0e Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Wed, 27 Jul 2022 12:43:20 +0200 Subject: [PATCH 08/16] Add update-verifier Change-Id: Id074d36297145e07f68d49be8f51feafcc726d5a --- upgrade-verifier/README | 9 +++++ upgrade-verifier/checker.sh | 81 +++++++++++++++++++++++++++++++++++++++++++++ upgrade-verifier/fvt.sh | 28 ++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 upgrade-verifier/README create mode 100755 upgrade-verifier/checker.sh create mode 100755 upgrade-verifier/fvt.sh diff --git a/upgrade-verifier/README b/upgrade-verifier/README new file mode 100644 index 0000000..8dff305 --- /dev/null +++ b/upgrade-verifier/README @@ -0,0 +1,9 @@ +1. Connect the target +2. Flash old image +3. Perform the upgrade using the delta file +4. Wait for the upgrade to complete +2. Run ./fvt.sh + +The fvt.sh script will copy checker.sh and update.cfg from the delta file by sdb and run checker.sh. +checker.sh will verify that the partitions checksum match. + diff --git a/upgrade-verifier/checker.sh b/upgrade-verifier/checker.sh new file mode 100755 index 0000000..4897313 --- /dev/null +++ b/upgrade-verifier/checker.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +CONFIG_PATH=update.cfg +BACKGROUND_COPY_LIST=/hal/etc/upgrade/background_copy.list + + +function is_cloned() { + [[ $(device_board_get_partition_ab_cloned) == "1" ]] +} + +function get_part_labels() { + echo -e "${1}_a\n${1}_b" +} + +function current_part_with_suffix() { + echo "${1}_$(current_suffix)" +} + +function opposite_part_with_suffix() { + echo "${1}_$(opposite_suffix)" +} + +function current_suffix() { + device_board_get_current_partition +} + +function opposite_suffix() { + if [[ $(current_suffix) == "a" ]]; then + echo "b" + else + echo "a" + fi +} + +function verify_part() { + PARTLABEL="$1" + SIZE="$2" + SHA="$3" + device=$(blkid -t PARTLABEL="$PARTLABEL" -o device) + CUR_SHA1=$(head -c "$SIZE" "$device" | sha1sum | awk '{print $1}') + [[ "$CUR_SHA1" == "$SHA" ]] +} + +function verify_update() { + echo -e "\n\nVerify update" + echo "Current slot: $(device_board_get_current_partition)" + while read -r LABEL_NAME DELTA_NAME TYPE DEV OFFSET OLD_SIZE NEW_SIZE OLD_SHA NEW_SHA + do + PARTLABEL=$(current_part_with_suffix "$LABEL_NAME") + echo -n "Checking $LABEL_NAME ($PARTLABEL): " + if verify_part "$PARTLABEL" "$NEW_SIZE" "$NEW_SHA"; then + echo "OK" + else + echo "Incorrect checksum" + fi + done < <(sed -e '/^#/d' -e '/^$/d' "$CONFIG_PATH") +} + +function verify_clone() { + echo -e "\n\nVerify clone" + echo "Opposite slot: $(opposite_suffix)" + while read -r LABEL_NAME + do + read -r SIZE SHA < <(awk "/$LABEL_NAME/ {print \$7 \" \" \$9}" $CONFIG_PATH) + PARTLABEL=$(opposite_part_with_suffix "$LABEL_NAME") + echo -n "Checking $LABEL_NAME ($PARTLABEL): " + if verify_part "$PARTLABEL" "$SIZE" "$SHA"; then + echo "OK" + else + echo "Incorrect checksum" + fi + + done < <(sed -e '/^#/d' -e '/^$/d' "$BACKGROUND_COPY_LIST") +} + +if ! is_cloned; then + clone_partitions.sh +fi + +verify_update +verify_clone diff --git a/upgrade-verifier/fvt.sh b/upgrade-verifier/fvt.sh new file mode 100755 index 0000000..49ef53b --- /dev/null +++ b/upgrade-verifier/fvt.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +DEST_DIR=/opt/usr/verifier +CHECKER_SH=checker.sh + + +if [ "$#" != "1" ] || [ ! -f "$1" ]; then + echo "Use:" + echo " $0 " + exit 1 +fi + +UPLOAD_FILES=( checker.sh update.cfg ) + +echo "Extract update.cfg from $1" +tar xvf "${1}" update.cfg + +echo "Upload files" +{ sleep 0.2; echo tizen; } | sdb shell su -c "mkdir -p $DEST_DIR && chmod 777 $DEST_DIR" +for FILE in "${UPLOAD_FILES[@]}"; do + sdb push "$FILE" "$DEST_DIR" +done + +{ sleep 0.2; echo tizen; } | sdb shell su -c "chmod +x $DEST_DIR/$CHECKER_SH" + +echo "Run verifier" +{ sleep 0.2; echo tizen; } | sdb shell su -c "cd $DEST_DIR && $DEST_DIR/$CHECKER_SH" + -- 2.7.4 From 2f95bbd34951462ad9087e7ea36c29a866f37da5 Mon Sep 17 00:00:00 2001 From: SangYoun Kwak Date: Thu, 18 Aug 2022 10:58:05 +0900 Subject: [PATCH 09/16] Modify upgrade-verifier to verify partitions with image files Change-Id: I7d83092a65ff153e18dd498e40122d6787318094 Signed-off-by: SangYoun Kwak --- upgrade-verifier/README | 7 +-- upgrade-verifier/checker.sh | 81 -------------------------- upgrade-verifier/fvt.sh | 28 --------- upgrade-verifier/upgrade-verifier-on-device.sh | 51 ++++++++++++++++ upgrade-verifier/upgrade-verifier.sh | 75 ++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 113 deletions(-) delete mode 100755 upgrade-verifier/checker.sh delete mode 100755 upgrade-verifier/fvt.sh create mode 100755 upgrade-verifier/upgrade-verifier-on-device.sh create mode 100755 upgrade-verifier/upgrade-verifier.sh diff --git a/upgrade-verifier/README b/upgrade-verifier/README index 8dff305..01b8632 100644 --- a/upgrade-verifier/README +++ b/upgrade-verifier/README @@ -2,8 +2,7 @@ 2. Flash old image 3. Perform the upgrade using the delta file 4. Wait for the upgrade to complete -2. Run ./fvt.sh - -The fvt.sh script will copy checker.sh and update.cfg from the delta file by sdb and run checker.sh. -checker.sh will verify that the partitions checksum match. +2. Run ./upgrade-verifier.sh +The upgrade-verifier.sh script will copy upgrade-verifier-on-device.sh and checksum.txt(created by upgrade-verifier.sh) by sdb and run upgrade-verifier-on-device.sh. +upgrade-verifier-on-device.sh will verify that the partitions checksum match. diff --git a/upgrade-verifier/checker.sh b/upgrade-verifier/checker.sh deleted file mode 100755 index 4897313..0000000 --- a/upgrade-verifier/checker.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -CONFIG_PATH=update.cfg -BACKGROUND_COPY_LIST=/hal/etc/upgrade/background_copy.list - - -function is_cloned() { - [[ $(device_board_get_partition_ab_cloned) == "1" ]] -} - -function get_part_labels() { - echo -e "${1}_a\n${1}_b" -} - -function current_part_with_suffix() { - echo "${1}_$(current_suffix)" -} - -function opposite_part_with_suffix() { - echo "${1}_$(opposite_suffix)" -} - -function current_suffix() { - device_board_get_current_partition -} - -function opposite_suffix() { - if [[ $(current_suffix) == "a" ]]; then - echo "b" - else - echo "a" - fi -} - -function verify_part() { - PARTLABEL="$1" - SIZE="$2" - SHA="$3" - device=$(blkid -t PARTLABEL="$PARTLABEL" -o device) - CUR_SHA1=$(head -c "$SIZE" "$device" | sha1sum | awk '{print $1}') - [[ "$CUR_SHA1" == "$SHA" ]] -} - -function verify_update() { - echo -e "\n\nVerify update" - echo "Current slot: $(device_board_get_current_partition)" - while read -r LABEL_NAME DELTA_NAME TYPE DEV OFFSET OLD_SIZE NEW_SIZE OLD_SHA NEW_SHA - do - PARTLABEL=$(current_part_with_suffix "$LABEL_NAME") - echo -n "Checking $LABEL_NAME ($PARTLABEL): " - if verify_part "$PARTLABEL" "$NEW_SIZE" "$NEW_SHA"; then - echo "OK" - else - echo "Incorrect checksum" - fi - done < <(sed -e '/^#/d' -e '/^$/d' "$CONFIG_PATH") -} - -function verify_clone() { - echo -e "\n\nVerify clone" - echo "Opposite slot: $(opposite_suffix)" - while read -r LABEL_NAME - do - read -r SIZE SHA < <(awk "/$LABEL_NAME/ {print \$7 \" \" \$9}" $CONFIG_PATH) - PARTLABEL=$(opposite_part_with_suffix "$LABEL_NAME") - echo -n "Checking $LABEL_NAME ($PARTLABEL): " - if verify_part "$PARTLABEL" "$SIZE" "$SHA"; then - echo "OK" - else - echo "Incorrect checksum" - fi - - done < <(sed -e '/^#/d' -e '/^$/d' "$BACKGROUND_COPY_LIST") -} - -if ! is_cloned; then - clone_partitions.sh -fi - -verify_update -verify_clone diff --git a/upgrade-verifier/fvt.sh b/upgrade-verifier/fvt.sh deleted file mode 100755 index 49ef53b..0000000 --- a/upgrade-verifier/fvt.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -DEST_DIR=/opt/usr/verifier -CHECKER_SH=checker.sh - - -if [ "$#" != "1" ] || [ ! -f "$1" ]; then - echo "Use:" - echo " $0 " - exit 1 -fi - -UPLOAD_FILES=( checker.sh update.cfg ) - -echo "Extract update.cfg from $1" -tar xvf "${1}" update.cfg - -echo "Upload files" -{ sleep 0.2; echo tizen; } | sdb shell su -c "mkdir -p $DEST_DIR && chmod 777 $DEST_DIR" -for FILE in "${UPLOAD_FILES[@]}"; do - sdb push "$FILE" "$DEST_DIR" -done - -{ sleep 0.2; echo tizen; } | sdb shell su -c "chmod +x $DEST_DIR/$CHECKER_SH" - -echo "Run verifier" -{ sleep 0.2; echo tizen; } | sdb shell su -c "cd $DEST_DIR && $DEST_DIR/$CHECKER_SH" - diff --git a/upgrade-verifier/upgrade-verifier-on-device.sh b/upgrade-verifier/upgrade-verifier-on-device.sh new file mode 100755 index 0000000..9bc2e5b --- /dev/null +++ b/upgrade-verifier/upgrade-verifier-on-device.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +function current_suffix() { + device_board_get_current_partition +} + +function opposite_suffix() { + if [[ $(current_suffix) == "a" ]]; then + echo "b" + else + echo "a" + fi +} + +function verify_partitions() { + local slot=$1 + local checksum_file=$2 + + echo -e "\n\nVerify partition ${slot}" + + while read -r label_name bin_name bin_size bin_sha1; do + local partlabel="${label_name}_${slot}" + local dev_path=$(blkid -t PARTLABEL="${partlabel}" -o device) + + echo -n "Checking ${label_name} (bin_name=${bin_name}, partlabel=${partlabel}, dev_path=${dev_path}): " + + if [[ -z ${dev_path} ]]; then + echo "Invalid partlabel" + continue + fi + + if [[ -z "${bin_name}" ]]; then + echo "No binary found, skip" + continue + fi + + local part_sha1=$(head -c ${bin_size} ${dev_path} | sha1sum | awk '{print $1}') + if [[ "${bin_sha1}" == "${part_sha1}" ]]; then + echo "OK" + else + echo "Incorrect checksum(${bin_sha1} != ${part_sha1})" + fi + done < <(sed -e '/^#/d' -e '/^$/d' "${checksum_file}") +} + +if [ "$(device_board_get_partition_ab_cloned)" != "1" ]; then + clone_partitions.sh +fi + +verify_partitions "$(current_suffix)" "checksums.txt" +verify_partitions "$(opposite_suffix)" "checksums.txt" diff --git a/upgrade-verifier/upgrade-verifier.sh b/upgrade-verifier/upgrade-verifier.sh new file mode 100755 index 0000000..04e7975 --- /dev/null +++ b/upgrade-verifier/upgrade-verifier.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +usage() { + echo "usage: $0 " +} + +create_checksum() { + local bl_img=$1 + local ap_img=$2 + local delta_cfg=$3 + local checksum_file=$4 + local temp_dir=".imgs_tmp" + + if [ -d ${temp_dir} ]; then + echo "${temp_dir} already exist" + return 1 + fi + + echo "Creating ${checksum_file}" + + mkdir ${temp_dir} + tar xfv ${bl_img} -C ${temp_dir} > /dev/null + tar xfv ${ap_img} -C ${temp_dir} > /dev/null + + : > ${checksum_file} + while read -r partition_label bin_name delta_name update_type blk_dev blk_offset; do + local bin_path="${temp_dir}/${bin_name}" + if [ ! -f ${bin_path} ]; then + echo "${bin_name} not exist" + echo "${partition_label}" >> ${checksum_file} + continue + fi + local checksum=$(sha1sum ${bin_path} | awk '{print $1}') + local bin_size=$(stat -c %s "${bin_path}") + echo "${partition_label} ${bin_name} ${bin_size} ${checksum}" >> ${checksum_file} + done < <(sed -e '/^#/d' -e '/^$/d' "${delta_cfg}") + + echo "${checksum_file} created" + rm ${temp_dir} -r + + return 0 +} + +# check parameters +if [ $# -ne 3 ]; then + usage $0 + exit 1 +fi + +BL_IMG=$1 +AP_IMG=$2 +DELTA_CFG=$3 +CHECKSUM_FILE="checksums.txt" +DEST_DIR="/opt/usr/verifier" +CHECKER_SCRIPT="upgrade-verifier-on-device.sh" + +if [ ! -f "${BL_IMG}" ] || [ ! -f "${AP_IMG}" ] || [ ! -f "${DELTA_CFG}" ]; then + usage $0 + exit 1 +fi + +create_checksum ${BL_IMG} ${AP_IMG} ${DELTA_CFG} ${CHECKSUM_FILE} +if [ $? -ne 0 ]; then + echo "failed to create checksum file" + exit 1 +fi + +echo "Upload files" +{ sleep 1; echo tizen; } | sdb shell su -c "mkdir -p $DEST_DIR && chmod 777 $DEST_DIR" +sdb push "${CHECKER_SCRIPT}" "${DEST_DIR}" +sdb push "${CHECKSUM_FILE}" "${DEST_DIR}" +{ sleep 1; echo tizen; } | sdb shell su -c "chmod +x $DEST_DIR/$CHECKER_SCRIPT" + +echo "Run verifier" +{ sleep 1; echo tizen; } | sdb shell su -c "cd $DEST_DIR && $DEST_DIR/$CHECKER_SCRIPT" -- 2.7.4 From c459a175da8efb5d3b379159badee86957d2e0d1 Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Tue, 23 Aug 2022 14:00:14 +0200 Subject: [PATCH 10/16] Update README after repository copy Copied platform/core/system/tota-upg -> upgrade-tools, which will handle delta generation from 7.5 onwards. Change-Id: I0413f75b634cb3907cf787a521ae415e95d36be3 --- README | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README b/README index 0ef4c2f..04819dd 100644 --- a/README +++ b/README @@ -1,7 +1,11 @@ - Tizen System Update - Tizen OTA Update Package Generator (tota-upg) +# Tizen System Update - Tizen System Update is the one of Tizen feature. This implements +This repository was previously known as platorm/core/system/tota-upg, +the "Tizen OTA Upgrade Package Generator". From Tizen 7.5 upgrade is +handled by platform/core/system/upgrade-tools repository. + + +Tizen System Update is the one of Tizen feature. This implements the functionality of firmware update base on OTA mechanism. * Notice of Limitations -- 2.7.4 From f75a5c888ab8b6a25653990d10cbe025bcef2c89 Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Wed, 24 Aug 2022 15:12:17 +0200 Subject: [PATCH 11/16] Import bsdiff from libtota This commit import tools to generate binary deltas from libtota commit 143447ad7 ("ss_bsdiff: Fix to speed up patch generation") Change-Id: I3ef5abd6065b476ffbd682c92bf306edd9a2ea4f --- LICENSE.Apache-2.0 | 202 ++++++++++++ LICENSE.BSD-2-Clause | 23 ++ bsdiff/CMakeLists.txt | 28 ++ bsdiff/ss_brotli_patch.c | 355 ++++++++++++++++++++ bsdiff/ss_brotli_patch.h | 22 ++ bsdiff/ss_bsdiff.c | 805 +++++++++++++++++++++++++++++++++++++++++++++ bsdiff/ss_bspatch.c | 155 +++++++++ bsdiff/ss_bspatch_common.c | 291 ++++++++++++++++ bsdiff/ss_bspatch_common.h | 17 + 9 files changed, 1898 insertions(+) create mode 100644 LICENSE.Apache-2.0 create mode 100644 LICENSE.BSD-2-Clause create mode 100644 bsdiff/CMakeLists.txt create mode 100644 bsdiff/ss_brotli_patch.c create mode 100644 bsdiff/ss_brotli_patch.h create mode 100644 bsdiff/ss_bsdiff.c create mode 100644 bsdiff/ss_bspatch.c create mode 100644 bsdiff/ss_bspatch_common.c create mode 100644 bsdiff/ss_bspatch_common.h diff --git a/LICENSE.Apache-2.0 b/LICENSE.Apache-2.0 new file mode 100644 index 0000000..fef8c29 --- /dev/null +++ b/LICENSE.Apache-2.0 @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/LICENSE.BSD-2-Clause b/LICENSE.BSD-2-Clause new file mode 100644 index 0000000..e9cc3cf --- /dev/null +++ b/LICENSE.BSD-2-Clause @@ -0,0 +1,23 @@ +Copyright 2003-2005 Colin Percival +Copyright 2012 Matthew Endsley +All rights reserved + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bsdiff/CMakeLists.txt b/bsdiff/CMakeLists.txt new file mode 100644 index 0000000..ecd7205 --- /dev/null +++ b/bsdiff/CMakeLists.txt @@ -0,0 +1,28 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.6) +PROJECT(ss_bsdiff C) + +SET(ss_bsdiff_SRCS ss_bsdiff.c) +SET(ss_bspatch_SRCS + ss_bspatch_common.c + ss_bspatch.c +) + +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/bsdiff) + +INCLUDE(FindPkgConfig) +pkg_check_modules(${PROJECT_NAME}_pkgs REQUIRED liblzma-tool libdivsufsort libbrotlienc) + +FOREACH(flag ${${PROJECT_NAME}_pkgs_CFLAGS}) + SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}") +ENDFOREACH(flag) + +SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} -I./include") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_CFLAGS}") + +ADD_EXECUTABLE(ss_bsdiff ${ss_bsdiff_SRCS}) +TARGET_LINK_LIBRARIES(ss_bsdiff ${${PROJECT_NAME}_pkgs_LDFLAGS} "-g" "-pthread") +INSTALL(TARGETS ss_bsdiff DESTINATION bin) + +ADD_EXECUTABLE(ss_bspatch ${ss_bspatch_SRCS}) +TARGET_LINK_LIBRARIES(ss_bspatch ${${PROJECT_NAME}_pkgs_LDFLAGS} "-g" "-pthread") +INSTALL(TARGETS ss_bspatch DESTINATION bin) diff --git a/bsdiff/ss_brotli_patch.c b/bsdiff/ss_brotli_patch.c new file mode 100644 index 0000000..6574d80 --- /dev/null +++ b/bsdiff/ss_brotli_patch.c @@ -0,0 +1,355 @@ +/* + * libtota + * + * Copyright (c) 2022 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fota_log.h" + +#define PF_OK 0 +#define PF_ERROR_OPEN_FILE 1 +#define PF_ERROR_MMAP 2 +#define PF_ERROR_INVALID_PATCH_FILE 3 +#define PF_ERROR_DECOMPRESSION 4 + +#define BUFF_IN_LEN 4096 +#define BUFF_OUT_LEN 4096 +#define SSINT_LEN 8 +#define BLOCK_COUNT_REPORT 5000 + +const char SSDIFF_MAGIC[] = "SSDIFF40"; + +struct bs_data { + int src_fd, dest_fd, patch_fd; + void *src_ptr, *dest_ptr, *patch_ptr; + size_t src_len, dest_len, patch_len; + unsigned char buff_in[BUFF_IN_LEN]; + unsigned char buff_out[BUFF_IN_LEN]; + uint8_t *dest_pos; + uint8_t *src_pos; + size_t available_in, available_out; + const uint8_t *compressed_pos; + uint8_t *decompressed_pos; + size_t total_size; + BrotliDecoderState *bstate; +}; + +static void free_data(struct bs_data *data) +{ + if (data == NULL) + return; + + if (data->src_ptr) munmap(data->src_ptr, data->src_len); + if (data->dest_ptr) munmap(data->dest_ptr, data->dest_len); + if (data->patch_ptr) munmap(data->patch_ptr, data->patch_len); + + if (data->src_fd) close(data->src_fd); + if (data->patch_fd) close(data->patch_fd); + if (data->dest_fd) close(data->dest_fd); +} + +static int open_file(char *file_name, int mode) +{ + assert(file_name); + int fd = open(file_name, mode, S_IWUSR | S_IRUSR); + if (fd < 0) + LOGE("Open file %s error: %m (%d)\n", file_name, errno); + return fd; +} + +static size_t get_file_len(int fd) +{ + assert(fd >= 0); + size_t result = lseek(fd, 0, SEEK_END); + lseek(fd, 0, SEEK_SET); + return result; +} + + +static size_t decompress_bytes(struct bs_data *data, size_t keep_offset) +{ + assert(data); + if (keep_offset > 0) { + memcpy(data->buff_out, data->buff_out + sizeof(data->buff_out) - keep_offset, keep_offset); + } + data->decompressed_pos = data->buff_out + keep_offset; + data->available_out = sizeof(data->buff_out) - keep_offset; + + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; + + result = BrotliDecoderDecompressStream(data->bstate, + &data->available_in, + &data->compressed_pos, + &data->available_out, + &data->decompressed_pos, + &data->total_size); + + if (result == BROTLI_DECODER_RESULT_ERROR) { + LOGE("Decoder error\n"); + return PF_ERROR_DECOMPRESSION; + } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + LOGE("Invalid source file\n"); + return PF_ERROR_DECOMPRESSION; + } + + return PF_OK; +} + +static int open_files(struct bs_data *data, char *source_file, size_t source_size, char *dest_file, size_t dest_size, char *patch_file) +{ + assert(data); + assert(source_file); + assert(dest_file); + assert(patch_file); + + data->src_fd = open_file(source_file, O_RDONLY); + data->patch_fd = open_file(patch_file, O_RDONLY); + data->dest_fd = open_file(dest_file, O_RDWR); + if (data->src_fd < 0 || + data->patch_fd < 0 || + data->dest_fd < 0) + return PF_ERROR_OPEN_FILE; + + data->src_len = source_size; + data->patch_len = get_file_len(data->patch_fd); + data->dest_len = dest_size; + + data->src_ptr = mmap(NULL, data->src_len, PROT_READ, MAP_PRIVATE, data->src_fd, 0); + if (data->src_ptr == MAP_FAILED) { + LOGE("mmap source file error: %m (%d)", errno); + return PF_ERROR_MMAP; + } + + data->patch_ptr = mmap(NULL, data->patch_len, PROT_READ, MAP_PRIVATE, data->patch_fd, 0); + if (data->patch_ptr == MAP_FAILED) { + LOGE("mmap patch file error: %m (%d)", errno); + return PF_ERROR_MMAP; + } + + data->dest_ptr = mmap(NULL, data->dest_len, PROT_WRITE, MAP_SHARED, data->dest_fd, 0); + if (data->dest_ptr == MAP_FAILED) { + LOGE("mmap destination error: %m (%d)\n", errno); + return PF_ERROR_MMAP; + } + + data->compressed_pos = data->patch_ptr; + data->available_in = data->patch_len; + + return PF_OK; +} + +static void init_data(struct bs_data *data) +{ + assert(data); + + data->src_fd = -1; + data->patch_fd = -1; + data->dest_fd = -1; + data->src_ptr = NULL; + data->dest_ptr = NULL; + data->patch_ptr = NULL; + data->src_len = 0; + data->dest_len = 0; + data->patch_len = 0; + data->available_in = 0; + data->compressed_pos = 0; + data->available_out = 0; + data->decompressed_pos = 0; + data->bstate = BrotliDecoderCreateInstance(NULL, NULL, NULL); +} + +static int64_t parse_ssint(unsigned char *buff) +{ + assert(buff); + /* + * From bsdiff 4.0 documentation: + * + * INTEGER type: + * + * offset size data type value + * 0 1 byte x0 + * 1 1 byte x1 + * 2 1 byte x2 + * 3 1 byte x3 + * 4 1 byte x4 + * 5 1 byte x5 + * 6 1 byte x6 + * 7 1 byte x7 + 128 * s + * + * The values x0, x2, x2, x3, x4, x5, x6 are between 0 and 255 (inclusive). + * The value x7 is between 0 and 127 (inclusive). The value s is 0 or 1. + * + * The INTEGER is parsed as: + * (x0 + x1 * 256 + x2 * 256^2 + x3 * 256^3 + x4 * 256^4 + + * x5 * 256^5 + x6 * 256^6 + x7 * 256^7) * (-1)^s + * + * (In other words, an INTEGER is a 64-byte signed integer in sign-magnitude + * format, stored in little-endian byte order.) + */ + int64_t result = *(int64_t*)buff & 0x7fffffff; + if ((buff[7] & 0x80) != 0) + result = -result; + + return result; +} + +int read_header(struct bs_data *data, uint8_t **buff_out_pos) +{ + assert(data); + assert(buff_out_pos); + + *buff_out_pos = data->buff_out; + + if (*buff_out_pos + sizeof(SSDIFF_MAGIC) > data->decompressed_pos || + memcmp(data->buff_out, SSDIFF_MAGIC, sizeof(SSDIFF_MAGIC) - 1) != 0) { + LOGE("Invalid patch file\n"); + return PF_ERROR_INVALID_PATCH_FILE; + } else { + LOGL(LOG_SSENGINE, "Looks like SSDIFF\n"); + } + + *buff_out_pos += sizeof(SSDIFF_MAGIC) - 1; + + if (*buff_out_pos + SSINT_LEN > data->decompressed_pos) { + decompress_bytes(data, data->decompressed_pos - *buff_out_pos); + *buff_out_pos = data->buff_out; + } + + size_t target_size = parse_ssint(*buff_out_pos); + LOGL(LOG_SSENGINE, "target_size: 0x%lx (%ld)\n", target_size, target_size); + + if (target_size != data->dest_len) { + LOGE("Declared target size differs from that read from the patch\n"); + return PF_ERROR_INVALID_PATCH_FILE; + } + + *buff_out_pos += SSINT_LEN; + + return PF_OK; +} + +int apply_patch_brotli(char *source_file, size_t source_size, char *dest_file, size_t dest_size, char *patch_file) +{ + assert(source_file); + assert(dest_file); + assert(patch_file); + + int result; + uint64_t blocks = 0; + struct bs_data data; + + init_data(&data); + + if ((result = open_files(&data, source_file, source_size, dest_file, dest_size, patch_file)) != PF_OK) + goto exit; + + if ((result = decompress_bytes(&data, 0)) != PF_OK) + goto exit; + + uint8_t *buff_out_pos; + + if ((result = read_header(&data, &buff_out_pos)) != PF_OK) + goto exit; + + uint64_t total_write = 0; + + while (total_write < data.dest_len) { + /* + * Make sure we can read the block header + */ + if (buff_out_pos + 4*8 > data.decompressed_pos) { + if ((result = decompress_bytes(&data, data.decompressed_pos - buff_out_pos)) != PF_OK) + goto exit; + buff_out_pos = data.buff_out; + } + + /* + * Read the block header + */ + int64_t diff_len = parse_ssint(buff_out_pos+0*8); + int64_t extra_len = parse_ssint(buff_out_pos+1*8); + int64_t old_pos = parse_ssint(buff_out_pos+2*8); + int64_t new_pos = parse_ssint(buff_out_pos+3*8); + buff_out_pos += 4*8; + + /* + * Prepare pointers + */ + data.dest_pos = data.dest_ptr + new_pos; + data.src_pos = data.src_ptr + old_pos; + /* + * Read diff data + */ + int64_t write = 0; + while (write < diff_len) { + if (buff_out_pos >= data.decompressed_pos) { + if ((result = decompress_bytes(&data, 0)) != PF_OK) + goto exit; + buff_out_pos = data.buff_out; + } + while (write < diff_len && buff_out_pos < data.decompressed_pos) { + *data.dest_pos = *(uint8_t*)(data.src_pos) + *(uint8_t*)buff_out_pos; + data.dest_pos++; + data.src_pos++; + buff_out_pos++; + write++; + } + } + total_write += write; + /* + * Read extra data + */ + write = 0; + while (write < extra_len) { + if (buff_out_pos >= data.decompressed_pos) { + if ((result = decompress_bytes(&data, 0)) != PF_OK) + goto exit; + buff_out_pos = data.buff_out; + } + int64_t chunk_size = extra_len - write; + if (buff_out_pos + chunk_size > data.decompressed_pos) { + chunk_size = data.decompressed_pos - buff_out_pos; + } + memcpy(data.dest_pos, buff_out_pos, chunk_size); + data.dest_pos += chunk_size; + buff_out_pos += chunk_size; + write += chunk_size; + } + total_write += write; + + blocks++; + if (blocks % BLOCK_COUNT_REPORT == 0) { + printf("Number of processed patch blocks: %lld\n", blocks); + } + } + + result = PF_OK; + +exit: + printf("Total processed blocks: %lld\n", blocks); + free_data(&data); + return result; +} diff --git a/bsdiff/ss_brotli_patch.h b/bsdiff/ss_brotli_patch.h new file mode 100644 index 0000000..47694b9 --- /dev/null +++ b/bsdiff/ss_brotli_patch.h @@ -0,0 +1,22 @@ +/* + * libtota + * + * Copyright (c) 2022 Samsung Electronics Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the License); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +extern int apply_patch_brotli(char *source_file, size_t source_size, char *dest_file, size_t dest_size, char *patch_file); diff --git a/bsdiff/ss_bsdiff.c b/bsdiff/ss_bsdiff.c new file mode 100644 index 0000000..76fbe41 --- /dev/null +++ b/bsdiff/ss_bsdiff.c @@ -0,0 +1,805 @@ +/*- + * Copyright 2003-2005 Colin Percival + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +// This file is a nearly copy of bsdiff.c from the +// bsdiff-4.3 distribution; the primary differences being how the +// input and output data are read, data post processing, +// search function modification and the error handling. + +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include +#include + +#define SUFSORT_MOD // Change suffix sorting algorithm from Qsufsort to Divsufsort +//#define ZLIB_MOD // Change compression algorithm +#define SEARCH_RECURSIVE // Change recursion of Search function to Iteration +//#define MAX_MATCH_SIZE // define ( MAX_MATCH_SIZE or CONST_MEMORY_USAGE ) or ( none of them ); use max_match memory for old file at patch side +#define CONST_MEMORY_USAGE 16384 // patch in m+O(1); m=size of new file; use only const memory for old file at patch side; +#define PATCH_FILE_FORMAT_MOD // no accumulation of diff and extra in db and eb; immediate write; also write all 3 parts of control stmt at same time +#define MULTI_THREADING 1 // only with #define CONST_MEMORY_USAGE or #define MAX_MATCH_SIZE +#define TIME_LIMIT_CHECK 4*60*60 // After TIME_LIMIT_CHECK seconds, new diff will be created with smaller score argument +#define TEMP_PATCH_NAME "temp_patch" +#define BROTLI_COMPRESSION_QUALITY 9 +#define ITERATIONS_COMPLETED 10 // After ITERATIONS_COMPLETED iterations elapsed time will be checked. Increasing it can cause the program to run longer than expected + // (the longer the program runs the longer the iterations) + +/* Take care : +1) Use either (MAX_MATCH_SIZE or CONST_MEMORY_USAGE) or (none of both). +2.) PATCH_FILE_FORMAT_MOD may or may not be used, independently of everything else. +3.) MULTI_THREADING can be used only with (CONST_MEMORY_USAGE or MAX_MATCH_SIZE). +*/ +#ifdef TIME_LIMIT_CHECK +long long outer_count = 0; +char ts1[256]; +void get_time_stamp(void) +{ + struct timeval tv; + int sec, msec; + + gettimeofday(&tv, NULL); + sec = (int)tv.tv_sec; + msec = (int)(tv.tv_usec / 1000); + snprintf(ts1, 256, "%06d.%03d", sec % 100000, msec); +} +#endif +#ifdef SUFSORT_MOD +//supporting only 32 bit divsufsort for now. +#include +#endif + +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) + +#ifdef MULTI_THREADING + +struct data_thread { + unsigned num_threads; + off_t size_thread; + + int fd; + int rc; + u_char *old; + u_char **new; + off_t oldsize; + off_t newsize; + + saidx_t *I; + u_char buf[8]; + u_char header[16]; + u_char buf2[32]; + + off_t lenn; + off_t lenn2; + + FILE * pf; + int bz2err; + FILE * pfbz2; +}; + +enum compression_method { + CM_LZMA, + CM_BROTLI, +}; + +struct bsdiff_info { + const char *old_file; + const char *new_file; + const char *patch_file; + enum compression_method comp_method; +}; + +struct data_thread data; + +int Function(int); + + +#endif + +static off_t matchlen(u_char *old, off_t oldsize, u_char *new, off_t newsize) +{ + off_t i; + for (i = 0; (i < oldsize) && (i < newsize); i++) + if (old[i] != new[i]) + break; + return i; +} + +static off_t search(saidx_t *I, u_char *old, off_t oldsize, + u_char *new, off_t newsize, off_t st, off_t en, off_t *pos) +{ + off_t x, y; + while (en - st >= 2) { + x = st + (en - st) / 2; + if (memcmp(old + I[x], new, MIN(oldsize - I[x], newsize)) < 0) + st = x; + else + en = x; + } + + x = matchlen(old + I[st], oldsize - I[st], new, newsize); + y = matchlen(old + I[en], oldsize - I[en], new, newsize); + + if (x > y) { + *pos = I[st]; + return x; + } else { + *pos = I[en]; + return y; + } +} + +static void offtout(off_t x, u_char *buf) +{ + off_t y; + + if (x < 0) + y = -x; + else + y = x; + + buf[0] = y % 256; + y -= buf[0]; + y = y / 256; + buf[1] = y % 256; + y -= buf[1]; + y = y / 256; + buf[2] = y % 256; + y -= buf[2]; + y = y / 256; + buf[3] = y % 256; + y -= buf[3]; + y = y / 256; + buf[4] = y % 256; + y -= buf[4]; + y = y / 256; + buf[5] = y % 256; + y -= buf[5]; + y = y / 256; + buf[6] = y % 256; + y -= buf[6]; + y = y / 256; + buf[7] = y % 256; + + if (x < 0) + buf[7] |= 0x80; +} + +static off_t bulk_read(int fd, u_char * buf, off_t size) +{ + off_t bytes_read = 0; + int ret = 0; + + while(bytes_read < size) + { + ret = read(fd, buf + bytes_read, size - bytes_read); + if (ret > 0) + bytes_read += ret; + else + break; + } + return bytes_read; +} + +int create_patch(const char *old_file, const char *new_file, const char *temp_patch, int offset_oldscore) +{ + assert(old_file); + assert(new_file); + + data.num_threads = MULTI_THREADING; + data.new = (u_char **)malloc(sizeof(u_char *)*data.num_threads); + + /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if (((data.fd = open(old_file, O_RDONLY, 0)) < 0) || + ((data.oldsize = lseek(data.fd, 0, SEEK_END)) == -1) || + ((data.old = malloc(data.oldsize + 1)) == NULL) || + (lseek(data.fd, 0, SEEK_SET) != 0) || + (bulk_read(data.fd, data.old, data.oldsize) != data.oldsize) || + (close(data.fd) == -1)) + err(1, "%s", old_file); + + data.I = malloc((data.oldsize + 1) * sizeof(saidx_t)); + if (!data.I) + err(1, "Memory allocation error: %s", old_file); + divsufsort(data.old, data.I, data.oldsize); + + /* Allocate newsize+1 bytes instead of newsize bytes to ensure + that we never try to malloc(0) and get a NULL pointer */ + if (((data.fd = open(new_file, O_RDONLY, 0)) < 0) || + ((data.newsize = lseek(data.fd, 0, SEEK_END)) == -1) || + (lseek(data.fd, 0, SEEK_SET) != 0)) + err(1, "%s", new_file); + data.size_thread = (data.newsize / data.num_threads); + + unsigned int j; + for (j = 0; j < data.num_threads; ++j) { + if (j != data.num_threads - 1) { + if (((data.new[j] = (u_char *)malloc(sizeof(u_char) * (data.size_thread + 1))) == NULL) || + (lseek(data.fd, 0, SEEK_CUR) != j * data.size_thread) || + (bulk_read(data.fd, data.new[j], data.size_thread) != data.size_thread)) + err(1, "%s", new_file); + } else { + if (((data.new[j] = (u_char *)malloc(sizeof(u_char) * (data.newsize - (j * data.size_thread) + 1))) == NULL) || + (lseek(data.fd, 0, SEEK_CUR) != j * data.size_thread) || + (bulk_read(data.fd, data.new[j], data.newsize - (j * data.size_thread)) != data.newsize - (j * data.size_thread))) + err(1, "here %s", new_file); + } + } + + if ((close(data.fd) == -1)) + err(1, "%s", new_file); + + /* Create the patch file */ + if ((data.pf = fopen(temp_patch, "w")) == NULL) + err(1, "%s", temp_patch); + + /* Header is + 0 8 "BSDIFF40" + 8 8 length of bzip2ed ctrl block + 16 8 length of bzip2ed diff block + 24 8 length of new file */ + /* File is + 0 32 Header + 32 ?? Bzip2ed ctrl block + ?? ?? Bzip2ed diff block + ?? ?? Bzip2ed extra block */ + memcpy(data.header, "SSDIFF40", 8); + + offtout(data.newsize, data.header + 8); + if (fwrite(data.header, 16, 1, data.pf) != 1) + err(1, "fwrite(%s)", temp_patch); + + /* Compute the differences, writing ctrl as we go */ + data.pfbz2 = data.pf; + //if ((data.pfbz2 = BZ2_bzWriteOpen(&data.bz2err, data.pf, 9, 0, 0)) == NULL) + //errx(1, "BZ2_bzWriteOpen, bz2err = %d", data.bz2err); + + + //BZ2_bzWriteClose(&data.bz2err, data.pfbz2, 0, NULL, NULL); + //if (data.bz2err != BZ_OK) + //errx(1, "BZ2_bzWriteClose, bz2err = %d", data.bz2err); + + int ret = Function(offset_oldscore); +#ifdef TIME_LIMIT_CHECK + if (ret != 0) { + printf("bsdiff fails to create delta with offset score %d\n", offset_oldscore); + printf("Old: [%s] -> New: [%s]\n", old_file, new_file); + } +#endif + /* Seek to the beginning, write the header, and close the file */ + if (fseeko(data.pf, 0, SEEK_SET)) + err(1, "fseeko"); + + if (fwrite(data.header, 16, 1, data.pf) != 1) + err(1, "fwrite(%s)", temp_patch); + if (fclose(data.pf)) + err(1, "fclose"); + /* Free the memory we used */ + free(data.I); + free(data.old); + free(data.new); + + return ret; +} + +int Function(int offset_oldscore) +{ + unsigned int thread_num = 0; + off_t end; + off_t scan = 0; + end = data.newsize; + int t1 = 0, t2 = 0; + get_time_stamp(); //total time capturing + t1 = atoi(ts1); + +#ifdef PATCH_FILE_FORMAT_MOD + u_char* db; + u_char* eb; + off_t dblen; + off_t eblen; +#endif + + off_t pos; + off_t len; + off_t lastscan; + off_t lastpos; + off_t lastoffset; + off_t oldscore; + off_t scsc; + off_t s; + off_t Sf; + off_t lenf; + off_t Sb; + off_t lenb; + off_t overlap; + off_t Ss; + off_t lens; + off_t i; + + pos = 0; + len = 0; + lastscan = 0; + lastpos = 0; + lastoffset = 0; + while (scan < end) { + oldscore = 0; + size_t prev_len; + uint64_t prev_pos, prev_oldscore; + int num_less_than_eight = 0; + for (scsc = scan += len; scan < end; scan++) { + prev_len = len; + prev_oldscore = oldscore; + prev_pos = pos; + len = search(data.I, data.old, data.oldsize, data.new[thread_num] + scan, end - scan, + len, data.oldsize, &pos); // Passing parameter as len instead of 0 for ramdisk.img etc taking long time + + for (; scsc < scan + len; scsc++) + if ((scsc + lastoffset < data.oldsize) && + (data.old[scsc + lastoffset] == data.new[thread_num][scsc])) + oldscore++; +#ifdef TIME_LIMIT_CHECK + if (offset_oldscore > 4) // when offset_oldscore is 4 and less we have to make sure diff is created no mater what, so we can't timeout + outer_count++; + if (outer_count > ITERATIONS_COMPLETED) { + outer_count = 0; + get_time_stamp(); //total time capturing + t2 = atoi(ts1); + //printf("\ntime diff = %d\n", (t2 - t1)); + if ((t2 - t1) > TIME_LIMIT_CHECK) + return 1; + } +#endif + if (((len == oldscore) && (len != 0)) || + (len > oldscore + offset_oldscore)) + break; + + if ((scan + lastoffset < data.oldsize) && + (data.old[scan + lastoffset] == data.new[thread_num][scan])) + oldscore--; + + /* + * Modification created by Thieu Le + * which speeds up patch generation time in situations + * where large blocks of data differ by less than 8 bytes. + * https://android.googlesource.com/platform/external/bsdiff/+/d172820cb8b4513478f0db5546d6e4d388adc1a7 + */ + const size_t fuzz = 8; + if (prev_len - fuzz <= len && len <= prev_len && + prev_oldscore - fuzz <= oldscore && + prev_pos <= pos && pos <= prev_pos + fuzz && + oldscore <= len && len <= oldscore + fuzz) { + num_less_than_eight++; + } else { + num_less_than_eight = 0; + } + if (num_less_than_eight > 100) break; + + }; + if ((len != oldscore) || (scan == end)) { + s = 0; + Sf = 0; + lenf = 0; + for (i = 0; (lastscan + i < scan) && (lastpos + i < data.oldsize); ) { + if (data.old[lastpos + i] == data.new[thread_num][lastscan + i]) + s++; + i++; + if (s * 2 - i > Sf * 2 - lenf) { + Sf = s; + lenf = i; + }; + }; + + lenb = 0; + if (scan < end) { + s = 0; + Sb = 0; + for (i = 1; (scan >= lastscan + i) && (pos >= i); i++) { + if (data.old[pos - i] == data.new[thread_num][scan - i]) + s++; + if (s * 2 - i > Sb * 2 - lenb) { + Sb = s; + lenb = i; + }; + }; + }; + + if (lastscan + lenf > scan - lenb) { + overlap = (lastscan + lenf) - (scan - lenb); + s = 0; + Ss = 0; + lens = 0; + for (i = 0; i < overlap; i++) { + if (data.new[thread_num][lastscan + lenf - overlap + i] == + data.old[lastpos + lenf - overlap + i]) + s++; + if (data.new[thread_num][scan - lenb + i] == + data.old[pos - lenb + i]) + s--; + if (s > Ss) { + Ss = s; + lens = i + 1; + }; + }; + + lenf += lens - overlap; + lenb -= lens; + }; + + if (((db = malloc(lenf + 1)) == NULL) || + ((eb = malloc((scan - lenb) - (lastscan + lenf) + 1)) == NULL)) + err(1, NULL); + + for (i = 0; i < lenf; i++) + db[i] = data.new[thread_num][lastscan + i] - data.old[lastpos + i]; + for (i = 0; i < (scan - lenb) - (lastscan + lenf); i++) + eb[i] = data.new[thread_num][lastscan + lenf + i]; + dblen = lenf; + eblen = (scan - lenb) - (lastscan + lenf); + offtout(lenf, data.buf2); + offtout((scan - lenb) - (lastscan + lenf), data.buf2 + 8); + offtout(lastpos, data.buf2 + 16); + offtout((data.size_thread * thread_num) + lastscan, data.buf2 + 24); + fwrite(data.buf2, 1, 32, data.pf); + //if (data.bz2err != BZ_OK) + //errx(1, "fwrite, bz2err = %d", data.bz2err); + + fwrite(db, 1, dblen, data.pf); + //if (data.bz2err != BZ_OK) + //errx(1, "fwrite, bz2err = %d", data.bz2err); + + fwrite(eb, 1, eblen, data.pf); + //if (data.bz2err != BZ_OK) + //errx(1, "fwrite, bz2err = %d", data.bz2err); + + free(db); + free(eb); + + lastscan = scan - lenb; + lastpos = pos - lenb; + lastoffset = pos - scan; + }; + }; + return 0; +} + +const char *kCantReadMessage = "Can not read input file"; +const char *kCantWriteMessage = "Can not write output file"; +const char *kCantAllocateMessage = "Can not allocate memory"; +const char *kDataErrorMessage = "Data error"; + +static void *SzAlloc(void *p, size_t size) +{ + p = p; + return MyAlloc(size); +} +static void SzFree(void *p, void *address) +{ + p = p; + MyFree(address); +} +static ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +int PrintError(char *buffer, const char *message, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError: %s\n", message); + return 1; +} + +int PrintErrorNumber(char *buffer, SRes val, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError code: %x\n", (unsigned)val); + return 1; +} + +int PrintUserError(char *buffer, int buf_size) +{ + return PrintError(buffer, "Incorrect command", buf_size); +} + +#define IN_BUF_SIZE (1 << 16) +#define OUT_BUF_SIZE (1 << 16) + + +static SRes lzma_encode(ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 fileSize, char *rs) +{ + CLzmaEncHandle enc; + SRes res; + CLzmaEncProps props; + + rs = rs; + + enc = LzmaEnc_Create(&g_Alloc); + if (enc == 0) + return SZ_ERROR_MEM; + + LzmaEncProps_Init(&props); + res = LzmaEnc_SetProps(enc, &props); + + if (res == SZ_OK) { + Byte header[LZMA_PROPS_SIZE + 8]; + size_t headerSize = LZMA_PROPS_SIZE; + int i; + + res = LzmaEnc_WriteProperties(enc, header, &headerSize); + for (i = 0; i < 8; i++) + header[headerSize++] = (Byte)(fileSize >> (8 * i)); + if (outStream->Write(outStream, header, headerSize) != headerSize) + res = SZ_ERROR_WRITE; + else { + if (res == SZ_OK) + res = LzmaEnc_Encode(enc, outStream, inStream, NULL, &g_Alloc, &g_Alloc); + } + } + LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc); + return res; +} + +int lzma_compress(const char *input_file, const char *output_file, char *rs, int rs_size) +{ + assert(input_file); + assert(output_file); + assert(rs); + + CFileSeqInStream inStream; + CFileOutStream outStream; + int res; + + FileSeqInStream_CreateVTable(&inStream); + File_Construct(&inStream.file); + + FileOutStream_CreateVTable(&outStream); + File_Construct(&outStream.file); + + size_t t4 = sizeof(UInt32); + size_t t8 = sizeof(UInt64); + if (t4 != 4 || t8 != 8) + return PrintError(rs, "Incorrect UInt32 or UInt64", rs_size); + + if (InFile_Open(&inStream.file, input_file) != 0) + return PrintError(rs, "Can not open input file", rs_size); + + + if (OutFile_Open(&outStream.file, output_file) != 0) + return PrintError(rs, "Can not open output file", rs_size); + + + UInt64 fileSize; + File_GetLength(&inStream.file, &fileSize); + res = lzma_encode(&outStream.s, &inStream.s, fileSize, rs); + + File_Close(&outStream.file); + File_Close(&inStream.file); + + if (res != SZ_OK) { + if (res == SZ_ERROR_MEM) + return PrintError(rs, kCantAllocateMessage, rs_size); + else if (res == SZ_ERROR_DATA) + return PrintError(rs, kDataErrorMessage, rs_size); + else if (res == SZ_ERROR_WRITE) + return PrintError(rs, kCantWriteMessage, rs_size); + else if (res == SZ_ERROR_READ) + return PrintError(rs, kCantReadMessage, rs_size); + return PrintErrorNumber(rs, res, rs_size); + } + return 0; +} + +int brotli_compress_internal(int input_fd, int output_fd, int quality) +{ + int res = -1; + size_t input_size = lseek(input_fd, 0, SEEK_END); + lseek(input_fd, 0, SEEK_SET); + void *input_file_ptr = mmap(NULL, input_size, PROT_READ, MAP_PRIVATE, input_fd, 0); + + if (input_file_ptr == MAP_FAILED) { + printf("Can not mmap input file: %d - %m\n", errno); + goto exit; + } + + BrotliEncoderState *bstate = BrotliEncoderCreateInstance(NULL, NULL, NULL); + if (bstate == 0) { + printf("Can not create BrotliEncoder instance\n"); + goto exit; + } + size_t max_output_size = BrotliEncoderMaxCompressedSize(input_size); + + if (max_output_size == 0) { + printf("Brotli engine error\n"); + goto exit; + } + + if (ftruncate(output_fd, max_output_size) == -1) { + printf("Can not truncate output file: %d - %m\n", errno); + goto exit; + } + + void *output_file_ptr = mmap(NULL, max_output_size, PROT_WRITE, MAP_SHARED, output_fd, 0); + if (output_file_ptr == MAP_FAILED) { + printf("Can not mmap output file: %d - %m\n", errno); + goto exit; + } + + if(!BrotliEncoderCompress(quality, + BROTLI_DEFAULT_WINDOW, + BROTLI_DEFAULT_MODE, + input_size, + input_file_ptr, + &max_output_size, + output_file_ptr)) { + printf("Compression error\n"); + goto exit; + } + if (ftruncate(output_fd, max_output_size) == -1) { + printf("Can not truncate output file after compression: %d - %m\n", errno); + goto exit; + } + + res = 0; +exit: + if (input_file_ptr) + munmap(input_file_ptr, input_size); + if (output_file_ptr) + munmap(output_file_ptr, max_output_size); + + return res; +} + +int brotli_compress(const char *input_file, const char *output_file, int quality) +{ + assert(input_file); + assert(output_file); + int res = -1; + + int input_fd = open(input_file, O_RDONLY); + if (input_fd < 0) { + printf("Can not open file: %s for read\n", input_file); + return res; + } + int output_fd = open(output_file, O_RDWR | O_CREAT, S_IWUSR | S_IRUSR); + if (output_fd < 0) { + printf("Can not open file: %s for write (%d: %m)\n", output_file, errno); + close(input_fd); + return res; + } + + res = brotli_compress_internal(input_fd, output_fd, quality); + + close(input_fd); + close(output_fd); + + return res; +} + +void print_help(const char *arg0) +{ + assert(arg0); + errx(1, "ss_bsdiff Version 5.0\nUsage: %s [-c ] oldfile newfile patchfile\n", arg0); +} + +int parse_args(struct bsdiff_info *info, int argc, char *argv[]) +{ + assert(info); + assert(argv); + + info->comp_method = CM_LZMA; // default compression method + + struct option long_options[] = { + {"compression", optional_argument, NULL, 'c'}, + {0, 0 , 0, 0} + }; + + int opt; + + while ((opt = getopt_long(argc, argv, "c:", long_options, NULL)) != -1) { + switch (opt) { + case 'c': + if (strcmp("lzma", optarg) == 0) + info->comp_method = CM_LZMA; + else if (strcmp("brotli", optarg) == 0) + info->comp_method = CM_BROTLI; + else { + err(1, "Unknown compression method: %s", optarg); + return -1; + } + } + } + + if (optind + 2 >= argc) { + err(1, "Not enough parameters"); + print_help(argv[0]); + return -1; + } + + info->old_file = argv[optind]; + info->new_file = argv[optind+1]; + info->patch_file = argv[optind+2]; + + return 0; +} + +int MY_CDECL main(int argc, char *argv[]) +{ + char rs[800] = { 0 }; + + struct bsdiff_info info; + if (parse_args(&info, argc, argv) != 0) + return 1; + + int ret = create_patch(info.old_file, info.new_file, TEMP_PATCH_NAME, 8); +#ifdef TIME_LIMIT_CHECK + if (ret != 0) { + /* + * Creating a patch with an offset score equal to 8 may take too long + * Therefore after a certain amount of time, patch creation is aborted + * and we run again with the score equal to 4. + */ + int score = 4; + printf("Trying with offset score %d\n", score); + ret = create_patch(info.old_file, info.new_file, TEMP_PATCH_NAME, score); + } + if (ret != 0) { + if (remove(TEMP_PATCH_NAME) < 0) + printf("Failed to remove %s\n", TEMP_PATCH_NAME); + err(1, "bsdiff fails to create delta within timelimit"); + } +#endif + int res = 0; + switch(info.comp_method) { + case CM_LZMA: + res = lzma_compress(TEMP_PATCH_NAME, info.patch_file, rs, sizeof(rs)); + break; + case CM_BROTLI: + res = brotli_compress(TEMP_PATCH_NAME, info.patch_file, BROTLI_COMPRESSION_QUALITY); + break; + default: + printf("Unknown compression method\n"); + res = -1; + break; + } + + if (remove(TEMP_PATCH_NAME) < 0) + printf("Failed to remove %s\n", TEMP_PATCH_NAME); + fputs(rs, stdout); + return res; +} diff --git a/bsdiff/ss_bspatch.c b/bsdiff/ss_bspatch.c new file mode 100644 index 0000000..8b6117f --- /dev/null +++ b/bsdiff/ss_bspatch.c @@ -0,0 +1,155 @@ +/*- + * Copyright 2003-2005 Colin Percival + * Copyright 2012 Matthew Endsley + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * Modifications are made in reimplementing suffix sort array generation + * and how the data is read and written to.Iterative part replaced the + * recursive implementation to avoid buffer overflow problems + */ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include + +#include +#include +#include + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include + +#include "ss_bspatch_common.h" + +const char *kCantReadMessage = "Can not read input file"; +const char *kCantWriteMessage = "Can not write output file"; +const char *kCantAllocateMessage = "Can not allocate memory"; +const char *kDataErrorMessage = "Data error"; + +int PrintError(char *buffer, const char *message, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError: %s\n", message); + return 1; +} + +int PrintErrorNumber(char *buffer, SRes val, int buf_size) +{ + snprintf(buffer + strlen(buffer), buf_size - strlen(buffer), + "\nError code: %x\n", (unsigned)val); + return 1; +} + +int PrintUserError(char *buffer, int buf_size) +{ + return PrintError(buffer, "Incorrect command", buf_size); +} + +int main2(int numArgs, const char *args[], char *rs, int rs_size) +{ + CFileSeqInStream inStream; + CFileOutStream outStream; + int res, fd; + int encodeMode; + unsigned char *buf_res = NULL; + unsigned char *new_data; + ssize_t new_size; + + FileSeqInStream_CreateVTable(&inStream); + File_Construct(&inStream.file); + + FileOutStream_CreateVTable(&outStream); + //File_Construct(&outStream.file); + + encodeMode = 0; + + size_t t4 = sizeof(UInt32); + size_t t8 = sizeof(UInt64); + if (t4 != 4 || t8 != 8) + return PrintError(rs, "Incorrect UInt32 or UInt64", rs_size); + + if (InFile_Open(&inStream.file, args[3]) != 0) + return PrintError(rs, "Can not open input file", rs_size); + + if (encodeMode) { + UInt64 fileSize; + File_GetLength(&inStream.file, &fileSize); + //res = Encode(&outStream.s, &inStream.s, fileSize, rs); + } else { + UInt64 unpackSize, i; + CLzmaDec state; + unsigned char header[LZMA_PROPS_SIZE + 8]; + RINOK(SeqInStream_Read(&inStream.s, header, sizeof(header))); + unpackSize = 0; + for (i = 0; i < 8; i++) + unpackSize += (UInt64)header[LZMA_PROPS_SIZE + i] << (i * 8); + buf_res = (unsigned char *)malloc(unpackSize); + if (!buf_res) + return PrintError(rs, "Failed to allocate memory", rs_size); + memset(buf_res, 0x0, unpackSize); + LzmaDec_Construct(&state); + RINOK(LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc)); + res = Decode2(&state, &outStream.s, &inStream.s, &unpackSize, buf_res); + LzmaDec_Free(&state, &g_Alloc); + File_Close(&inStream.file); + if (apply_patch(args[1], buf_res, &new_data, &new_size) != 0) { + if (new_data) + free(new_data); + return 1; + } + if (((fd = open(args[2], O_CREAT | O_TRUNC | O_WRONLY, 0666)) < 0) || + (write(fd, new_data, new_size) != new_size) || (close(fd) == -1)) + err(1, "%s", args[2]); + if (res != SZ_OK) { + free(new_data); + free(buf_res); + if (res == SZ_ERROR_MEM) + return PrintError(rs, kCantAllocateMessage, rs_size); + else if (res == SZ_ERROR_DATA) + return PrintError(rs, kDataErrorMessage, rs_size); + else if (res == SZ_ERROR_WRITE) + return PrintError(rs, kCantWriteMessage, rs_size); + else if (res == SZ_ERROR_READ) + return PrintError(rs, kCantReadMessage, rs_size); + return PrintErrorNumber(rs, res, rs_size); + } + free(new_data); + free(buf_res); + return 0; + } + return 1; +} + +int main(int numArgs, const char *args[]) +{ + char rs[800] = { 0 }; + if (numArgs != 4) + errx(1, "ss_bspatch Version 1.0\nUsage: ss_bspatch oldfile newfile patchfile\n"); + int res = main2(numArgs, args, rs, sizeof(rs)); + fputs(rs, stdout); + return res; +} diff --git a/bsdiff/ss_bspatch_common.c b/bsdiff/ss_bspatch_common.c new file mode 100644 index 0000000..e7bd45e --- /dev/null +++ b/bsdiff/ss_bspatch_common.c @@ -0,0 +1,291 @@ +/*- + * Copyright 2003-2005 Colin Percival + * Copyright 2012 Matthew Endsley + * All rights reserved + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted providing that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * Modifications are made in reimplementing suffix sort array generation + * and how the data is read and written to.Iterative part replaced the + * recursive implementation to avoid buffer overflow problems + */ +//#define ZLIB_MOD //not stable yet. +//#define MAX_MATCH_SIZE // define ( MAX_MATCH_SIZE or CONST_MEMORY_USAGE ) or ( none of them ) +#define CONST_MEMORY_USAGE (64*1024) //tests show smallest time when using 64 kb +#define PATCH_FILE_FORMAT_MOD +#define BSDIFF_HEADER "BSDIFF40" +#define SSDIFF_HEADER "SSDIFF40" +//#define MULTI_THREADING +#include +#include +#include + +#include +#include +#include + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include + +static void *SzAlloc(void *p, size_t size) +{ + p = p; + return MyAlloc(size); +} + +static void SzFree(void *p, void *address) +{ + p = p; + MyFree(address); +} +ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +static off_t offtin(u_char *buf) +{ + off_t y; + + y = buf[7] & 0x7F; + y = y * 256; + y += buf[6]; + y = y * 256; + y += buf[5]; + y = y * 256; + y += buf[4]; + y = y * 256; + y += buf[3]; + y = y * 256; + y += buf[2]; + y = y * 256; + y += buf[1]; + y = y * 256; + y += buf[0]; + + if (buf[7] & 0x80) + y = -y; + + return y; +} + +#define IN_BUF_SIZE (1 << 16) +#define OUT_BUF_SIZE (1 << 16) + +SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, + UInt64 *unpackSize, unsigned char *dec_data) +{ + int thereIsSize = (*unpackSize != (UInt64)(Int64) - 1); + UInt64 offset = 0; + Byte inBuf[IN_BUF_SIZE]; + Byte outBuf[OUT_BUF_SIZE]; + size_t inPos = 0, inSize = 0, outPos = 0; + + LzmaDec_Init(state); + + offset = 0; + + for (;;) { + if (inPos == inSize) { + inSize = IN_BUF_SIZE; + RINOK(inStream->Read(inStream, inBuf, &inSize)); + inPos = 0; + } + + SRes res; + SizeT inProcessed = inSize - inPos; + SizeT outProcessed = OUT_BUF_SIZE - outPos; + ELzmaFinishMode finishMode = LZMA_FINISH_ANY; + ELzmaStatus status; + + if (thereIsSize && outProcessed > *unpackSize) { + outProcessed = (SizeT) * unpackSize; + finishMode = LZMA_FINISH_END; + } + + res = LzmaDec_DecodeToBuf(state, outBuf + outPos, &outProcessed, + inBuf + inPos, &inProcessed, finishMode, &status); + inPos += inProcessed; + outPos += outProcessed; + *unpackSize -= outProcessed; + memcpy(dec_data + offset, outBuf, outProcessed); + offset += outProcessed; + + outPos = 0; + + if ((res != SZ_OK) || (thereIsSize && *unpackSize == 0)) + return res; + + if (inProcessed == 0 && outProcessed == 0) { + if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK) + return SZ_ERROR_DATA; + return res; + } + } +} + +int apply_patch(const char *oldfile, unsigned char *patch_buffer, unsigned char **dest_buf, ssize_t *dest_size) +{ + int fd = -1, result = 0; + off_t oldsize, newsize; + u_char header[16], buf[8]; + u_char *old = NULL; + off_t oldpos, newpos; + off_t ctrl[4]; /////////////////////////////////////THREAD + off_t total_write; /////////////////////////////////////////THREAD + off_t j; + off_t memory_usage = CONST_MEMORY_USAGE; + off_t match_size; + off_t patch_buffer_offset = 0; + bool flag; + + /* + File format: + 0 8 "BSDIFF40" + 8 8 X + 16 8 Y + 24 8 sizeof(newfile) + 32 X bzip2(control block) + 32+X Y bzip2(diff block) + 32+X+Y ??? bzip2(extra block) + with control block a set of triples (x,y,z) meaning "add x bytes + from oldfile to x bytes from the diff block; copy y bytes from the + extra block; seek forwards in oldfile by z bytes". + */ + // Read header + if (patch_buffer) + memcpy(header, patch_buffer, 16); + else { + printf("%s().%d Corrupt decoded patch buffer\n", __FUNCTION__, __LINE__); + return 1; + } + + /* Check for appropriate magic */ + if (memcmp(header, BSDIFF_HEADER, 8) != 0 && memcmp(header, SSDIFF_HEADER, 8) != 0) { + printf("%s().%d Patch buffer header corrupt\n", __FUNCTION__, __LINE__); + return 1; + } + + /* Read lengths from header */ + newsize = offtin(header + 8); + + if ((newsize < 0)) { + printf("%s().%d Patch buffer corrupt\n", __FUNCTION__, __LINE__); + return 1; + } + + /* Cset patch_buffer_offset at the right place */ + patch_buffer_offset += 16; + + if (((fd = open(oldfile, O_RDONLY, 0)) < 0) || + ((oldsize = lseek(fd, 0, SEEK_END)) == -1) || + ((old = malloc(memory_usage + 1)) == NULL) || + (lseek(fd, 0, SEEK_SET) != 0)) { + printf("Corruption in old file %s\n", oldfile); + result = 1; + goto Cleanup; + } + + if ((*dest_buf = malloc(newsize + 1)) == NULL) { + printf("Corruption in old file %s\n", oldfile); + result = 1; + goto Cleanup; + } + oldpos = 0; + newpos = 0; + + total_write = 0; + + while (total_write != newsize) { + /* Read control data */ + for (j = 0; j <= 3; j++) { + memcpy(buf, patch_buffer + patch_buffer_offset, 8); + patch_buffer_offset += 8; + ctrl[j] = offtin(buf); + }; + + total_write += (ctrl[0] + ctrl[1]); + newpos = ctrl[3]; + oldpos = ctrl[2]; + + ////////////////////////////////////////////////////////////////////////////////// + flag = true; + match_size = ctrl[0]; + while (flag == true) { + if (match_size <= memory_usage) { + if (pread(fd, old, match_size, oldpos) != match_size) { + printf("Corruption in old file %s\n", oldfile); + result = 1; + goto Cleanup; + } + if (newpos + match_size > newsize) { + printf("%s().%d Corrupt patch\n", __FUNCTION__, __LINE__); + result = 1; + goto Cleanup; + } + memcpy((*dest_buf) + newpos, patch_buffer + patch_buffer_offset, match_size); + patch_buffer_offset += match_size; + for (j = 0; j < match_size; j++) + (*dest_buf)[newpos + j] += old[j]; + newpos += match_size; + flag = false; + } else { + if (pread(fd, old, memory_usage, oldpos) != memory_usage) { + printf("%s().%d Corruption in old file %s\n", __FUNCTION__, __LINE__ , oldfile); + result = 1; + goto Cleanup; + } + if (newpos + memory_usage > newsize) { + printf("%s().%d Corrupt patch\n", __FUNCTION__, __LINE__); + result = 1; + goto Cleanup; + } + memcpy((*dest_buf) + newpos, patch_buffer + patch_buffer_offset, memory_usage); + patch_buffer_offset += memory_usage; + for (j = 0; j < memory_usage; j++) + (*dest_buf)[newpos + j] += old[j]; + match_size -= memory_usage; + oldpos += memory_usage; + newpos += memory_usage; + } + } + + //////////////////////////////////////////////////////////////////////////////////////// + /* Sanity-check */ + if (newpos + ctrl[1] > newsize) { + printf("%s().%d Corrupt patch\n", __FUNCTION__, __LINE__); + result = 1; + goto Cleanup; + } + /* Read extra string */ + memcpy((*dest_buf) + newpos, patch_buffer + patch_buffer_offset, ctrl[1]); + patch_buffer_offset += ctrl[1]; + }; + *dest_size = newsize; +Cleanup: + //close old file + if (fd >= 0) + close(fd); + if (old) + free(old); + return result; +} diff --git a/bsdiff/ss_bspatch_common.h b/bsdiff/ss_bspatch_common.h new file mode 100644 index 0000000..3075ea2 --- /dev/null +++ b/bsdiff/ss_bspatch_common.h @@ -0,0 +1,17 @@ +#ifndef _SS_BSPATCH_COMMON_H +#define _SS_BSPATCH_COMMON_H 1 + +#include +#include <7zFile.h> +#include <7zVersion.h> +#include +#include + +extern ISzAlloc g_Alloc; + +SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, + UInt64 *unpackSize, unsigned char *dec_data); + +int apply_patch(const char *oldfile, unsigned char *patch_buffer, unsigned char **dest_buf, ssize_t *dest_size); + +#endif /* _SS_BSPATCH_COMMON_H */ -- 2.7.4 From 2648c176f156718c9f83effa65ef82c1367a0a5e Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Wed, 24 Aug 2022 15:58:26 +0200 Subject: [PATCH 12/16] Add dockerfile to provide upgrade-tools in reproductible manner Build with: ./docker-build.sh Run with: docker run --rm -ti --privileged -v /dev:/dev -v /data:/data upgrade-tools:latest Update volume (-v) mappings as needed. Note that --privileged and -v /dev:/dev are needed for loop device mounting to work correctly. /data volume is just example how to transfer tizen images into container and deltas - out of container. Change-Id: I5bd09d446df32eda44dddffb9dcc5bee942cd427 --- Dockerfile | 21 +++++++++++++++++++++ docker-build.sh | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 Dockerfile create mode 100755 docker-build.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ce810e1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:20.04 as build + +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ=UTC + +ADD bsdiff /bsdiff + +RUN apt-get update && apt-get -y --no-install-recommends install libbrotli-dev libdivsufsort-dev git cmake build-essential pkg-config +RUN git clone git://git.tizen.org/platform/upstream/lzma-sdk -b tizen /lzma-sdk +RUN cd /lzma-sdk && cmake -DLIB_INSTALL_DIR=/usr/local/lib . && make install +RUN cd bsdiff && cmake . && make install + +FROM ubuntu:20.04 + +ADD mk_delta /tota-upg/mk_delta/ +ADD scripts /tota-upg/scripts/ +ADD recovery /tota-upg/recovery/ +COPY --from=build /usr/local/bin/ss_bsdiff /usr/local/bin/ss_bspatch /usr/local/bin/ +COPY --from=build /usr/local/lib/liblzma-tool.so.* /usr/local/lib +RUN apt-get update && \ + apt-get install -y --no-install-recommends libbrotli1 libdivsufsort3 python-is-python2 python2 python-apt p7zip-full attr tar file sudo && rm -rf /var/lib/apt/lists/* diff --git a/docker-build.sh b/docker-build.sh new file mode 100755 index 0000000..b6f6486 --- /dev/null +++ b/docker-build.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec docker build --no-cache -t upgrade-tools:latest . -- 2.7.4 From 7bf407772a158c50f8bb5835d4c6067d506dcb3f Mon Sep 17 00:00:00 2001 From: Mateusz Moscicki Date: Wed, 24 Aug 2022 17:17:49 +0200 Subject: [PATCH 13/16] ss_bsdiff: Change the search function This commit uses the provided sa_search() function to look for patterns, instead of using own implementation. The change is inspired by: https://android.googlesource.com/platform/external/bsdiff/+/refs/heads/master/suffix_array_index.cc Change-Id: I49b3c4dd12c11b81157e030851bb8e4e48d1f6b2 --- bsdiff/ss_bsdiff.c | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/bsdiff/ss_bsdiff.c b/bsdiff/ss_bsdiff.c index 76fbe41..d892de2 100644 --- a/bsdiff/ss_bsdiff.c +++ b/bsdiff/ss_bsdiff.c @@ -146,23 +146,24 @@ static off_t matchlen(u_char *old, off_t oldsize, u_char *new, off_t newsize) static off_t search(saidx_t *I, u_char *old, off_t oldsize, u_char *new, off_t newsize, off_t st, off_t en, off_t *pos) { - off_t x, y; - while (en - st >= 2) { - x = st + (en - st) / 2; - if (memcmp(old + I[x], new, MIN(oldsize - I[x], newsize)) < 0) - st = x; - else - en = x; - } + off_t x,y; + int left = 0; - x = matchlen(old + I[st], oldsize - I[st], new, newsize); - y = matchlen(old + I[en], oldsize - I[en], new, newsize); + int count = sa_search(old, oldsize, new, newsize, I, en, &left); + if (count > 0) { + *pos = I[left]; + return newsize; + } - if (x > y) { - *pos = I[st]; + if (left > 0) { + x = matchlen(old + I[left - 1], oldsize - I[left - 1], new, newsize); + } + y = matchlen(old + I[left], oldsize - I[left], new, newsize); + if(left > 0 && x > y) { + *pos = I[left - 1]; return x; } else { - *pos = I[en]; + *pos = I[left]; return y; } } @@ -370,11 +371,10 @@ int Function(int offset_oldscore) prev_oldscore = oldscore; prev_pos = pos; len = search(data.I, data.old, data.oldsize, data.new[thread_num] + scan, end - scan, - len, data.oldsize, &pos); // Passing parameter as len instead of 0 for ramdisk.img etc taking long time + 0, data.oldsize, &pos); - for (; scsc < scan + len; scsc++) - if ((scsc + lastoffset < data.oldsize) && - (data.old[scsc + lastoffset] == data.new[thread_num][scsc])) + for (; scsc < scan + len && scsc + lastoffset < data.oldsize; scsc++) + if (data.old[scsc + lastoffset] == data.new[thread_num][scsc]) oldscore++; #ifdef TIME_LIMIT_CHECK if (offset_oldscore > 4) // when offset_oldscore is 4 and less we have to make sure diff is created no mater what, so we can't timeout @@ -405,9 +405,10 @@ int Function(int offset_oldscore) const size_t fuzz = 8; if (prev_len - fuzz <= len && len <= prev_len && prev_oldscore - fuzz <= oldscore && + oldscore <= prev_oldscore && prev_pos <= pos && pos <= prev_pos + fuzz && oldscore <= len && len <= oldscore + fuzz) { - num_less_than_eight++; + num_less_than_eight++; } else { num_less_than_eight = 0; } -- 2.7.4 From 62df3bcb33693f255e002ad49a6b0c5d9bb32762 Mon Sep 17 00:00:00 2001 From: Karol Lewandowski Date: Fri, 26 Aug 2022 12:27:32 +0200 Subject: [PATCH 14/16] Drop unused delta.ua from image delta.ua is legacy style upgrade agent that is no longer supported. Change-Id: Ib5c17736dc91643e05d90a0d40f3c69edea2bd75 --- mk_delta/common/bin/mk_delta.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/mk_delta/common/bin/mk_delta.sh b/mk_delta/common/bin/mk_delta.sh index dc15cce..1aad474 100755 --- a/mk_delta/common/bin/mk_delta.sh +++ b/mk_delta/common/bin/mk_delta.sh @@ -364,7 +364,6 @@ do echo "Extract binaries for update from images" fn_extract_from_image rootfs.img /usr/libexec/upgrade-support fn_extract_from_image rootfs.img /usr/bin/upgrade-apply - fn_extract_from_image ramdisk-recovery.img /usr/bin/delta.ua #--- archive result directory --- cd result/$MONDATE -- 2.7.4 From 0a49080f797018268c9bcc28806cd6242d5c7478 Mon Sep 17 00:00:00 2001 From: Antoni Adaszkiewicz Date: Mon, 29 Aug 2022 14:19:06 +0200 Subject: [PATCH 15/16] CreatePatch.py: Add support for hardlinks during delta generation. Change-Id: Ie3fd13d9557f36222d96c61db1ce4b54cb5e2d82 --- mk_delta/common/bin/CreatePatch.py | 190 ++++++++++++++++++++++--------------- 1 file changed, 114 insertions(+), 76 deletions(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index 43e7f14..99399df 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -82,6 +82,7 @@ NEW_FILES_ZIP_NAME = "system.7z" SYMLINK_TYPE = "SYM" ATTR_DOC_EXT = "_attr.txt" SYMLINK_DOC_NAME = "_sym.txt" +HARDLINK_DOC_NAME = "_hard.txt" PART_DOC_EXT = ".txt" DIFF_PREFIX = "diff" DIFF_SUFFIX = ".delta" @@ -330,7 +331,7 @@ def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE): file_out.write(line + '\n') -def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes): +def Update_Attr(RequestedPath, Type, File_Attributes, Sym_Attributes): # Full File Path should MATCH if GenerateDiffAttr == "FALSE": return @@ -340,9 +341,9 @@ def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes): for line in f: if FilePath in line: if Type == SYMLINK_TYPE: - Sym_Attibutes.append(line) + Sym_Attributes.append(line) else: - File_Attibutes.append(line) + File_Attributes.append(line) def hash_file(filename): @@ -364,50 +365,14 @@ def hash_file(filename): return h.hexdigest() -def find_dupes_dir(BASE_OLD, BASE_NEW): - dups = {} - fdupes = {} - print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW)) - logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW)) - for rootbase, subdirsB, fileListB in os.walk(BASE_OLD): - #print('Scanning %s...' % rootbase) - for filename in fileListB: - path = os.path.join(rootbase, filename) - if os.path.islink(path): - continue - # Calculate hash - file_hash = hash_file(path) - dups[file_hash] = path - - for roottarget, subdirsT, fileListT in os.walk(BASE_NEW): - #print('Scanning %s...' % roottarget) - for filename in fileListT: - # Get the path to the file - path = os.path.join(roottarget, filename) - if os.path.islink(path): - continue - # Calculate hash - file_hash = hash_file(path) - # Add or append the file path - if file_hash in dups: - BaseStr = dups.get(file_hash) - Baseloc = path.find('/') - TarLoc = BaseStr.find('/') - if not path[Baseloc:] == BaseStr[TarLoc:]: - logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:])) - fdupes[path] = BaseStr - logging.info('Total Duplicate files %d' % (len(fdupes))) - return fdupes - - -def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT): +def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT, Old_hardlinks, New_hardlinks): dups = {} fdupes = {} print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW)) for filename in fileListB: Src_File = BASE_OLD + '/' + filename - if os.path.islink(Src_File) or os.path.isdir(Src_File): + if os.path.islink(Src_File) or os.path.isdir(Src_File) or ishardlink(Src_File): continue # Calculate hash file_hash = hash_file(Src_File) @@ -415,7 +380,7 @@ def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT): for filename in fileListT: Dest_File = BASE_NEW + '/' + filename - if os.path.islink(Dest_File) or os.path.isdir(Dest_File): + if os.path.islink(Dest_File) or os.path.isdir(Dest_File) or ishardlink(Dest_File): continue # Calculate hash file_hash = hash_file(Dest_File) @@ -425,7 +390,6 @@ def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT): if not BaseStr[Baseloc:] == filename: #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename)) fdupes[BaseStr] = filename - logging.info('Total Duplicate files %d' % (len(fdupes))) return fdupes @@ -456,14 +420,15 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi files_changed = [] files_unchanged = [] files_renamed = [] - File_Attibutes = [] - Sym_Attibutes = [] + File_Attributes = [] + Sym_Attributes = [] files_Del_List = {} files_New_List = {} - MyDict_Patches = {} - PWD = os.getcwd() + # Get dictionaries used for hardlinks form both directories + New_hardlinks = get_hardlinks(BASE_NEW) + Old_hardlinks = get_hardlinks(BASE_OLD) # Generate NEW List for elt in New_files: @@ -498,14 +463,20 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi dst_file = BASE_NEW + '/' + elt #print('Files Changed - %s -%s' % (src_file,dst_file)) if os.path.islink(src_file) and os.path.islink(dst_file): - if not os.readlink(src_file) == os.readlink(dst_file): + if not (os.readlink(src_file) == os.readlink(dst_file)): files_changed.append(elt) #print('%d Sym link files changed' % len(files_changed)) logging.info('Sym links Changed - %s' % elt) else: files_unchanged.append(elt) - # Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink) - elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file): + # Both are hardlinks - we add them because we can't be sure if file they point to changes + elif elt in New_hardlinks and elt in Old_hardlinks: + files_changed.append(elt) + # Both are Normal files and they differ. (Is file returns true in case of sym/hardlink also, + # so additional check to find either of the file is sym/hardlink) + elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) \ + and (not (elt in New_hardlinks or elt in Old_hardlinks)) \ + and os.path.isfile(src_file) and os.path.isfile(dst_file): if not filecmp.cmp(src_file, dst_file): files_changed.append(elt) #print('%d Normal files changed' % len(files_changed)) @@ -561,9 +532,10 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi # this file is the same name in both! src_file = BASE_OLD + '/' + value dst_file = BASE_NEW + '/' + files_New_List[key] - olddirpath = path_head(files_New_List[key]) - newdirpath = path_head(value) - if os.path.islink(src_file) or os.path.islink(dst_file): + # we don't want to move hardlinks + if ishardlink(src_file) or ishardlink(dst_file): + logging.debug('Cannot diff as one of them is a hardlink') + elif os.path.islink(src_file) or os.path.islink(dst_file): logging.debug('Cannot diff as one of them is Symlink') elif os.path.isdir(src_file) or os.path.isdir(dst_file): logging.debug('Cannot diff as one of them is dir') @@ -580,14 +552,18 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi ''' Sym_Diff_Cnt = 0 Sym_New_Cnt = 0 + Hard_Diff_Cnt = 0 + Hard_New_Cnt = 0 Del_Cnt = 0 New_Cnt = 0 Diff_Cnt = 0 Move_Cnt = 0 Verbatim_Cnt = 0 SymLinkDoc = OUT_DIR + '/' + PART_NAME + SYMLINK_DOC_NAME + HardLinkDoc = OUT_DIR + '/' + PART_NAME + HARDLINK_DOC_NAME Partition_Doc = open(OUT_DIR + '/' + PART_NAME + '.txt', 'w') Partition_Doc_SymLinks = open(SymLinkDoc, 'w') + Partition_Doc_HardLinks = open(HardLinkDoc, "w") print("writing diff'ed changed files...") for elt in files_changed: @@ -600,9 +576,18 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi patch = os.readlink(dst_file) Sym_Diff_Cnt = Sym_Diff_Cnt + 1 Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch)) - Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes) + Update_Attr(elt, "SYM", File_Attributes, Sym_Attributes) + # Both are hardlinks and they differ (point to something different, new/changed file) + if elt in Old_hardlinks and elt in New_hardlinks: + if Old_hardlinks[elt] != New_hardlinks[elt] or New_hardlinks[elt] in files_changed or New_hardlinks[elt] in files_new: + logging.debug('Hardlinks changed %s %s' % (src_file, dst_file)) + patch = New_hardlinks[elt] + Hard_Diff_Cnt += 1 + Partition_Doc_HardLinks.write('HARD:DIFF:%s:%s:%s\n' % (elt, elt, patch)) # Both are NORMAL files and they differ - elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file): + elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) \ + and (not (elt in Old_hardlinks or elt in New_hardlinks)) \ + and os.path.isfile(dst_file) and os.path.isfile(src_file): # Both are files and they differ Diff_Cnt = Diff_Cnt + 1 patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt)) @@ -619,17 +604,16 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi else: Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName)) - Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes) + Update_Attr(elt, "FILE", File_Attributes, Sym_Attributes) # Both differ but they are of diff types else: # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here files_removed.append(elt) files_new.append(elt) - fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new) + fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new, Old_hardlinks, New_hardlinks) for oldpath, newpath in fdupes.iteritems(): logging.info('Dupes %s -> %s' % (oldpath, newpath)) - for elt in files_removed: src_file = BASE_OLD + '/' + elt # If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE) @@ -640,7 +624,6 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file))) files_removed.remove(elt) files_new.remove(fdupes[src_file]) - # Should be placed after removing duplicates, else they will be filtered here. # loop shd b for all NEW files, rather than for all delete files (Current understanding) # First Step: Sort & Filter out unwanted files @@ -650,7 +633,6 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi # 3. File name length shd b greater than 3 char # 4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break. # 5. Should consider editdistance for RENAME LOGIC ==> TBD - Base_DelList = files_removed[:] Base_NewList = files_new[:] DelList = sorted(Base_DelList, key=path_leaf) @@ -663,6 +645,8 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi for file in DelList: if os.path.islink(BASE_OLD + '/' + file): continue + elif ishardlink(BASE_OLD + '/' + file): + continue elif os.path.isdir(BASE_OLD + '/' + file): continue else: @@ -674,6 +658,8 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi for file in NewList: if os.path.islink(BASE_NEW + '/' + file): continue + elif ishardlink(BASE_NEW + '/' + file): + continue elif os.path.isdir(BASE_NEW + '/' + file): continue elif len(path_leaf(file)) <= 3: @@ -681,9 +667,7 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi continue else: Filter2.append(file) - NewList = Filter2 - logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList))) for new_file in NewList: @@ -713,7 +697,7 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi winning_patch_sz = DiffSize winning_file = del_file elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and R_Flag == 'TRUE'): - logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file)) + logging.debug('Because nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file)) break if len(winning_file) > 0: logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize)) @@ -740,8 +724,11 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi if os.path.isdir(src_file) or os.path.isdir(dst_file): # This case never occurs?? Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1])) - Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes) - else: # Make sure these files are PROPER and they shd NOT be symlinks + Update_Attr(elt[0], "FILE", File_Attributes, Sym_Attributes) + # Make sure these files are PROPER and they shd NOT be symlinks + elif not (os.path.islink(src_file) or os.path.islink(dst_file)) \ + and not (elt[0] in New_hardlinks or elt[1] in Old_hardlinks) \ + and (os.path.isfile(src_file) and os.path.isfile(dst_file)): if filecmp.cmp(src_file, dst_file): Move_Cnt = Move_Cnt + 1 Diff_Cnt = Diff_Cnt - 1 @@ -757,7 +744,7 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName)) SS_UpdateSize(src_file, dst_file) - Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes) + Update_Attr(elt[0], "FILE", File_Attributes, Sym_Attributes) # HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality. # So that, the rename functionality will NOT create PATCH instead of verbatims. @@ -777,6 +764,8 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi src_file = BASE_OLD + '/' + elt if os.path.islink(src_file): Partition_Doc.write('DEL:SYM:%s\n' % (elt)) + elif elt in Old_hardlinks: + Partition_Doc.write('DEL:HARD:%s\n' % (elt)) elif os.path.isdir(src_file): # If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed. Partition_Doc.write('DEL:REG:%s:NA\n' % (elt)) @@ -813,8 +802,17 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi if not os.path.exists(path_head(destpath)): os.makedirs(path_head(destpath)) logging.info('New SymLink - Adding missing Dir') - #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes) + Update_Attr(elt, "SYM", File_Attributes, Sym_Attributes) Sym_New_Cnt = Sym_New_Cnt + 1 + elif elt in New_hardlinks: + patch = New_hardlinks[elt] + logging.debug('File new hardlink %s' % elt) + Partition_Doc_HardLinks.write('HARD:NEW:%s:%s\n' %(elt, patch)) + destpath = newfiles_dest_path + elt + if not os.path.exists(path_head(destpath)): + os.makedirs(path_head(destpath)) + logging.info('New hardlink - Adding missing Dir') + Hard_New_Cnt += 1 elif os.path.isdir(dst_file): # We create just empty directory here destpath = newfiles_dest_path + elt if not os.path.exists(destpath): @@ -841,6 +839,7 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi except Exception as exc: logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath)) raise exc + Update_Attr(elt, "FILE", File_Attributes, Sym_Attributes) for elt in Dir_Added: newfiles_dest_path = 'run/upgrade-sysroot/' @@ -873,29 +872,39 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi logging.info('%d files unchanged' % len(files_unchanged)) logging.info('%d files files_renamed' % len(files_renamed)) logging.info('%d files NEW' % len(files_new)) - logging.info('%d File attr' % len(File_Attibutes)) - logging.info('%d Sym attr' % len(Sym_Attibutes)) - logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt)) - print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt)) + logging.info('%d File attr' % len(File_Attributes)) + logging.info('%d Sym attr' % len(Sym_Attributes)) + logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d HardDiffs-%d HardNews-%d Verbatim -%d\n' % \ + (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Hard_Diff_Cnt, Hard_New_Cnt, Verbatim_Cnt)) + print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d HardDiffs-%d HardNews-%d Verbatim -%d\n' % \ + (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Hard_Diff_Cnt, Hard_New_Cnt, Verbatim_Cnt)) # There could be duplicates, TODO, can check before adding.. ATTR_FILE_D = open(ATTR_FILE, 'a+') - for elt in File_Attibutes: + for elt in File_Attributes: ATTR_FILE_D.write(elt) - for elt in Sym_Attibutes: + for elt in Sym_Attributes: ATTR_FILE_D.write(elt) ATTR_FILE_D.close() Partition_Doc_SymLinks.close() + Partition_Doc_HardLinks.close() Partition_Read_SymLinks = open(SymLinkDoc, 'r+') + Partition_Read_HardLinks = open(HardLinkDoc, 'r+') Partition_Doc.write(Partition_Read_SymLinks.read()) - Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt)) - Partition_Doc_SymLinks.close() + for line in reversed(Partition_Read_HardLinks.readlines()): + Partition_Doc.write(line) + Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d %d %d\n' % \ + (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Hard_Diff_Cnt, Hard_New_Cnt)) + Partition_Read_SymLinks.close() + Partition_Read_HardLinks.close() Partition_Doc.close() os.remove(SymLinkDoc) + os.remove(HardLinkDoc) - if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0: + if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + Hard_Diff_Cnt + \ + Hard_New_Cnt + os.path.getsize(ATTR_FILE) == 0: print('No Delta Generated for %s - %s' % (PART_NAME, OUT_DIR)) logging.info('No Delta Generated for %s' % PART_NAME) shutil.rmtree(OUT_DIR) @@ -920,12 +929,41 @@ def NewFiles(src, dest): def measure_two_filediffs(src, dst): patchLoc = 'temp.patch' + # TODO ensure this is excepts an error subprocess.call([DIFF_UTIL, src, dst, patchLoc]) result_size = os.path.getsize(patchLoc) os.remove(patchLoc) return result_size +def ishardlink(path): + if os.stat(path).st_nlink > 1: + return True + return False + + +def get_inode(path): + return os.stat(path).st_ino + + +def get_hardlinks(base): + hardlinks_dict = {} + inodes_dict = {} + + for root, direcotories, files in os.walk(base, topdown=True, followlinks=False): + for file in sorted(files): + file_name = os.path.join(root, file) + if not os.path.islink(file_name) and ishardlink(file_name): + inode = get_inode(file_name) + rel_path = os.path.relpath(file_name, base) + if inode not in inodes_dict: + inodes_dict[inode] = rel_path + else: + hardlinks_dict[rel_path] = inodes_dict[inode] + + return hardlinks_dict + + def Get_Files(path): all_files = [] all_dirs = [] -- 2.7.4 From 8285576c0122ccb4e12546d877a59f80dca19a9b Mon Sep 17 00:00:00 2001 From: Antoni Adaszkiewicz Date: Tue, 6 Sep 2022 15:31:42 +0200 Subject: [PATCH 16/16] Change DELTA_FS type deltas to be created with relative pathnames and remove their dependency on /run/upgrade-sysroot directory Change-Id: I58348748e5359047bd587a6a7cf34af570665940 --- mk_delta/common/bin/CreatePatch.py | 54 +++++++++++++----------------------- mk_delta/common/bin/mk_part_delta.sh | 34 +++++++++-------------- 2 files changed, 33 insertions(+), 55 deletions(-) diff --git a/mk_delta/common/bin/CreatePatch.py b/mk_delta/common/bin/CreatePatch.py index 99399df..f8dacb1 100755 --- a/mk_delta/common/bin/CreatePatch.py +++ b/mk_delta/common/bin/CreatePatch.py @@ -6,7 +6,6 @@ import filecmp import shutil import subprocess import re -import ntpath import zipfile import datetime import hashlib @@ -76,7 +75,7 @@ COMMON_BIN_PATH = "../../common/bin/" DIFF_UTIL = "/usr/local/bin/ss_bsdiff" DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch" #ZIPUTIL = "p7zip " -ZIPUTIL = "7z -mf=off a system.7z " +ZIPUTIL = "7z -mf=off a " NEW_FILES_PATH = "run/upgrade-sysroot" NEW_FILES_ZIP_NAME = "system.7z" SYMLINK_TYPE = "SYM" @@ -305,15 +304,10 @@ def ensure_dir_exists(path): def path_leaf(path): - head, tail = ntpath.split(path) # This is for windows?? Recheck + head, tail = os.path.split(path) return tail -def path_head(path): - head, tail = ntpath.split(path) - return head - - # Creating Diff between OLD and NEW attribute files v12 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE): if GenerateDiffAttr == "FALSE": @@ -335,7 +329,7 @@ def Update_Attr(RequestedPath, Type, File_Attributes, Sym_Attributes): # Full File Path should MATCH if GenerateDiffAttr == "FALSE": return - FilePath = '"/' + RequestedPath + '"' + FilePath = '"' + RequestedPath + '"' #print ('FilePath - %s'% (FilePath)) with open(AttributeFile) as f: for line in f: @@ -672,7 +666,6 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi for new_file in NewList: R_Flag = 'FALSE' - DirPathNew = path_head(new_file) FileNameNew = path_leaf(new_file) DiffSize = 0 winning_patch_sz = os.path.getsize(BASE_NEW + '/' + new_file) @@ -785,22 +778,23 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Del_Cnt = Del_Cnt + 1 logging.debug(' Dir Deleted- %s' % src_file) + try: + ensure_dir_exists(NEW_FILES_PATH) + except FileExistsError as exc: + logging.error('Directory %s used by this script is already an existing file' % NEW_FILES_PATH) + raise exc + for elt in files_new: - dst_file = BASE_NEW + '/' + elt - newfiles_dest_path = 'run/upgrade-sysroot/' - try: - ensure_dir_exists(newfiles_dest_path) - except FileExistsError as exc: - logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path) - raise exc + dst_file = os.path.join(BASE_NEW, elt) + destpath = os.path.join(NEW_FILES_PATH, elt) + if os.path.islink(dst_file): patch = os.readlink(dst_file) logging.debug(' File New Links %s' % elt) Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch)) # What if this is only a new sym link and folder already exists??? Should recheck - destpath = newfiles_dest_path + elt - if not os.path.exists(path_head(destpath)): - os.makedirs(path_head(destpath)) + if not os.path.exists(os.path.dirname(destpath)): + os.makedirs(os.path.dirname(destpath)) logging.info('New SymLink - Adding missing Dir') Update_Attr(elt, "SYM", File_Attributes, Sym_Attributes) Sym_New_Cnt = Sym_New_Cnt + 1 @@ -808,20 +802,17 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi patch = New_hardlinks[elt] logging.debug('File new hardlink %s' % elt) Partition_Doc_HardLinks.write('HARD:NEW:%s:%s\n' %(elt, patch)) - destpath = newfiles_dest_path + elt - if not os.path.exists(path_head(destpath)): - os.makedirs(path_head(destpath)) + if not os.path.exists(os.path.dirname(destpath)): + os.makedirs(os.path.dirname(destpath)) logging.info('New hardlink - Adding missing Dir') Hard_New_Cnt += 1 elif os.path.isdir(dst_file): # We create just empty directory here - destpath = newfiles_dest_path + elt if not os.path.exists(destpath): os.makedirs(destpath) logging.debug(' File New Dir %s' % destpath) New_Cnt = New_Cnt + 1 else: New_Cnt = New_Cnt + 1 - destpath = newfiles_dest_path + elt destdir = os.path.dirname(destpath) logging.debug('New files - %s ==> %s' % (dst_file, destdir)) @@ -842,13 +833,7 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi Update_Attr(elt, "FILE", File_Attributes, Sym_Attributes) for elt in Dir_Added: - newfiles_dest_path = 'run/upgrade-sysroot/' - try: - ensure_dir_exists(newfiles_dest_path) - except FileExistsError as exc: - logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path) - raise exc - destpath = newfiles_dest_path + elt + destpath = os.path.join(NEW_FILES_PATH, elt) if not os.path.exists(destpath): os.makedirs(destpath) logging.debug(' DirList New Dir %s' % destpath) @@ -858,9 +843,10 @@ def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_fi print 'Compressing New files' if (New_Cnt > 0 or Sym_New_Cnt > 0): WorkingDir = os.getcwd() - os.chdir(os.getcwd() + "/" + NEW_FILES_PATH) + os.chdir(os.path.join(os.getcwd(), NEW_FILES_PATH)) logging.info('Curr Working Dir - %s' % os.getcwd()) - os.system(ZIPUTIL + NEW_FILES_PATH + " >> " + LOGFILE) + log_path = os.path.join(WorkingDir, LOGFILE) + os.system(ZIPUTIL + NEW_FILES_ZIP_NAME + " . " + " >> " + log_path) shutil.move(NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR) # New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression SS_UpdateSize(WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME) diff --git a/mk_delta/common/bin/mk_part_delta.sh b/mk_delta/common/bin/mk_part_delta.sh index a3ad1bf..3f1f49e 100755 --- a/mk_delta/common/bin/mk_part_delta.sh +++ b/mk_delta/common/bin/mk_part_delta.sh @@ -94,13 +94,13 @@ fn_mk_attribute() echo "Attribute generation for ${TARGET} [START] $(date +%T)" - sudo find ./${BASE_DIR_OLD} -type f -printf '"/%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; > ${V1_ATTR_FILE} - sudo find ./${BASE_DIR_OLD} -type l -printf '"/%P" SymLink 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V1_ATTR_FILE} - sudo find ./${BASE_DIR_OLD} -type d -printf '"/%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V1_ATTR_FILE} + sudo find ./${BASE_DIR_OLD} -type f -printf '"%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; > ${V1_ATTR_FILE} + sudo find ./${BASE_DIR_OLD} -type l -printf '"%P" SymLink 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V1_ATTR_FILE} + sudo find ./${BASE_DIR_OLD} -type d -printf '"%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V1_ATTR_FILE} - sudo find ./${BASE_DIR_NEW} -type f -printf '"/%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; > ${V2_ATTR_FILE} - sudo find ./${BASE_DIR_NEW} -type l -printf '"/%P" SymLink 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V2_ATTR_FILE} - sudo find ./${BASE_DIR_NEW} -type d -printf '"/%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V2_ATTR_FILE} + sudo find ./${BASE_DIR_NEW} -type f -printf '"%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; > ${V2_ATTR_FILE} + sudo find ./${BASE_DIR_NEW} -type l -printf '"%P" SymLink 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V2_ATTR_FILE} + sudo find ./${BASE_DIR_NEW} -type d -printf '"%P" Regular 14 %04m:%04U:%04G:' -exec ${COMMON_BINDIR}/${ATTR_CMD} {} \; >> ${V2_ATTR_FILE} ################ Change user and group permission from '0' to '0000' ############ sed -i 's/: 0/:0000/g' ${V1_ATTR_FILE} @@ -114,10 +114,11 @@ fn_mk_attribute() sed -i 's/: /:0/g' ${V2_ATTR_FILE} ################ Change ":./old" to "/" ############ - sed -i "s/:.\/${BASE_DIR_OLD}\//:\//" ${V1_ATTR_FILE} - sed -i "s/:.\/${BASE_DIR_OLD}/:\//" ${V1_ATTR_FILE} - sed -i "s/:.\/${BASE_DIR_NEW}\//:\//" ${V2_ATTR_FILE} - sed -i "s/:.\/${BASE_DIR_NEW}/:\//" ${V2_ATTR_FILE} + sed -i "s/:.\/${BASE_DIR_OLD}\//:/" ${V1_ATTR_FILE} + sed -i "s/:.\/${BASE_DIR_OLD}/:/" ${V1_ATTR_FILE} + + sed -i "s/:.\/${BASE_DIR_NEW}\//:/" ${V2_ATTR_FILE} + sed -i "s/:.\/${BASE_DIR_NEW}/:/" ${V2_ATTR_FILE} echo "Attribute generation for ${TARGET} [END] $(date +%T)" } @@ -394,7 +395,6 @@ fn_mk_delta_fs() DEBUG_DELTA=debug_${DELTA} #CFG_XML=${PART_NAME}_FS.xml - FAKE_ROOT=run/upgrade-sysroot BASE_OLD=${PART_NAME}_OLD BASE_NEW=${PART_NAME}_NEW @@ -402,32 +402,24 @@ fn_mk_delta_fs() case "${PART_NAME}" in "rootfs" ) EXCLUDE_FILES="lost+found dev proc tmp var sys csa" - TGT_MNT_PNT=${FAKE_ROOT} - ;; "ramdisk1" | "ramdisk" ) EXCLUDE_FILES="lost+found dev proc tmp sys" - TGT_MNT_PNT=${FAKE_ROOT}/mnt/initrd ;; "ramdisk2" | "ramdisk-recovery" ) EXCLUDE_FILES="lost+found" - TGT_MNT_PNT=${FAKE_ROOT}/mnt/initrd-recovery ;; "user" ) EXCLUDE_FILES="lost+found" - TGT_MNT_PNT=${FAKE_ROOT}/opt/usr ;; "system-data" ) EXCLUDE_FILES="lost+found" - TGT_MNT_PNT=${FAKE_ROOT}/opt ;; "kernel" | "boot" ) EXCLUDE_FILES="" - TGT_MNT_PNT=${FAKE_ROOT}/boot ;; "hal" ) EXCLUDE_FILES="" - TGT_MNT_PNT="${FAKE_ROOT}/hal" ;; * ) @@ -436,8 +428,8 @@ fn_mk_delta_fs() ;; esac - MNT_PNT_OLD=${BASE_OLD}/${TGT_MNT_PNT} - MNT_PNT_NEW=${BASE_NEW}/${TGT_MNT_PNT} + MNT_PNT_OLD=${BASE_OLD}/ + MNT_PNT_NEW=${BASE_NEW}/ fn_gen_metadata fn_mk_delta_fs_core -- 2.7.4