21 if sys.hexversion < 0x02040000:
22 print >> sys.stderr, "Python 2.4 or newer is required."
26 Diff two folders and create delta using SS_BSDIFF
27 Will maintain same format of script that will be generated when we use diffutil
29 1. Create a list of files in each Base folders,
30 2. These files will fall into one these below categories:
31 1) Only in OLD - Should be deleted
32 2) Only in NEW - Should be added or renamed accordingly
33 3) File exists in both directories but contents are different - Create Diff.
34 4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
35 5) Duplicates in the list of Deletes and News
36 6) Close matching diffs even though name changes across directories. (for matching extension)
37 7) Clearing empty directories after Moves or diffs under Rename.
38 8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
41 1. Given two folders, from list of REMOVED and NEW files find if there
42 is version change and create diff between them
45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
46 Catching errors at all stages. SHOULD exit & return error in case of failure
54 global NEW_FILES_ZIP_NAME
57 global SYMLINK_DOC_NAME
63 global SUPPORT_CONTAINERS
68 global COMMON_BIN_PATH
75 COMMON_BIN_PATH = "../../common/bin/"
76 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
77 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
79 ZIPUTIL = "7z -mf=off a system.7z "
80 NEW_FILES_PATH = "run/upgrade-sysroot"
81 NEW_FILES_ZIP_NAME = "system.7z"
83 ATTR_DOC_EXT = "_attr.txt"
84 SYMLINK_DOC_NAME = "_sym.txt"
87 DIFF_SUFFIX = ".delta"
89 FULL_IMAGE = "FULL_IMAGE"
90 DELTA_IMAGE = "DELTA_IMAGE"
94 VERBATIM_LIST = "Verbatim_List.txt"
98 COMPRESSION_LZMA = "lzma"
99 COMPRESSION_BROTLI = "brotli"
101 SUPPORT_RENAME = "TRUE" # Use appropriate name
102 SUPPORT_CONTAINERS = "FALSE"
103 SUPPORT_VERBATIM = "TRUE"
109 logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
111 global GenerateDiffAttr
114 if len(sys.argv) < 5:
115 sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
116 UPDATE_TYPE = sys.argv[1]
117 UPDATE_TYPE_S = UPDATE_TYPE.split(":")
118 PART_NAME = sys.argv[2] # lets make this also optional
120 BASE_OLD = sys.argv[3]
121 BASE_NEW = sys.argv[4]
122 OUT_DIR = sys.argv[5]
125 UPDATE_CFG_PATH = EMPTY
126 GenerateDiffAttr = "FALSE"
127 if UPDATE_TYPE_S[0] == DELTA_FS:
128 #instead of arguments check it in outdirectory ?
129 if len(sys.argv) == 9:
130 ATTR_OLD = sys.argv[6]
131 ATTR_NEW = sys.argv[7]
132 UPDATE_CFG_PATH = '../' + sys.argv[8]
133 GenerateDiffAttr = "TRUE"
135 elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]:
136 if len(sys.argv) == 7:
137 #Use path in better way
138 UPDATE_CFG_PATH = '../' + sys.argv[6]
141 global DIFFPATCH_UTIL
142 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
143 DIFF_UTIL = COMMON_BIN_PATH + DIFF_UTIL
144 DIFFPATCH_UTIL = COMMON_BIN_PATH + DIFFPATCH_UTIL
145 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
146 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
147 logging.info('Diff Util Does NOT exist -- ABORT')
150 start = datetime.datetime.now().time()
151 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
152 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s] [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]' % (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
154 ensure_dir_exists(OUT_DIR)
155 if GenerateDiffAttr == "TRUE":
156 if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
157 print >> sys.stderr, "Attributes missing -- ABORT"
160 # Should check if APT is supported on other linux flavours
162 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
163 logging.info('Basic utils installed')
165 print >> sys.stderr, "Basic utils missing -- ABORT"
168 if UPDATE_TYPE_S[0] == FULL_IMAGE:
169 SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
170 # #### currently does not support LZMA ####
171 # elif UPDATE_TYPE == DELTA_IMAGE:
172 # SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
173 elif UPDATE_TYPE_S[0] == DELTA_IMAGE:
174 SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
175 elif UPDATE_TYPE == DELTA_FS:
176 AttributeFile = ATTR_NEW
177 ATTR_FILE = OUT_DIR + '/' + PART_NAME + ATTR_DOC_EXT
178 Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
179 Old_files, Old_dirs = Get_Files(BASE_OLD)
180 New_files, New_dirs = Get_Files(BASE_NEW)
181 SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
183 if not UPDATE_CFG_PATH == EMPTY:
184 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
186 elif UPDATE_TYPE == EXTRA:
187 print('UPDATE_TYPE ---- EXTRA')
189 print('UPDATE_TYPE ---- UNKNOWN FORMAT')
191 if GenerateDiffAttr == "TRUE":
192 if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
195 end = datetime.datetime.now().time()
197 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
198 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
199 logging.info('Time start [%s] - Time end [%s]' % (start, end))
200 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
202 except Exception as exc:
203 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
207 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
208 f = open(UPDATE_CFG_PATH, 'r')
209 lines = f.readlines()
211 f = open(UPDATE_CFG_PATH, 'w')
213 ConfigItems = line.split()
214 if ConfigItems[0] == DELTA_BIN:
215 DELTA = ConfigItems[1]
216 logging.info('Updating %s config' % DELTA_BIN)
217 line = line.rstrip('\n')
219 line = line.replace(line, line + '\t' + str(Value) + '\n')
226 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
229 oldsize_d = os.path.getsize(BASE_OLD)
230 newsize_d = os.path.getsize(BASE_NEW)
231 SHA_BIN_DEST = hash_file(BASE_NEW)
232 SHA_BIN_BASE = hash_file(BASE_OLD)
234 #incase UPDATE CFG is empty
236 SS_UpdateSize(BASE_OLD, BASE_NEW)
237 #Should throw error if PART NAME NOT found??
238 if not UPDATE_CFG_PATH == EMPTY:
239 f = open(UPDATE_CFG_PATH, 'r')
240 lines = f.readlines()
242 f = open(UPDATE_CFG_PATH, 'w')
244 ConfigItems = line.split()
245 if ConfigItems[0] == DELTA_BIN:
246 logging.info('Updating %s config' % DELTA_BIN)
247 DELTA = ConfigItems[1]
248 line = line.rstrip('\n')
249 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
255 patchLoc = '%s/%s' % (OUT_DIR, DELTA)
256 logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW, DELTA_BIN, patchLoc))
257 subprocess.call([DIFF_UTIL, "-c", COMPRESSION_METHOD, BASE_OLD, BASE_NEW, patchLoc])
260 def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
261 logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW, DELTA_BIN))
262 oldsize_d = os.path.getsize(BASE_OLD)
263 newsize_d = os.path.getsize(BASE_NEW)
264 SHA_BIN_DEST = hash_file(BASE_NEW)
265 SHA_BIN_BASE = hash_file(BASE_OLD)
266 #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
267 SS_UpdateSize(BASE_OLD, BASE_NEW)
269 if not UPDATE_CFG_PATH == EMPTY:
270 f = open(UPDATE_CFG_PATH, 'r')
271 lines = f.readlines()
273 f = open(UPDATE_CFG_PATH, 'w')
275 ConfigItems = line.split()
276 if ConfigItems[0] == DELTA_BIN:
277 logging.info('Updating %s config' % DELTA_BIN)
278 DELTA = ConfigItems[1]
279 line = line.rstrip('\n')
280 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
287 def zipdir(path, zip):
288 for root, dirs, files in os.walk(path):
290 zip.write(os.path.join(root, file))
293 def ensure_dir_exists(path):
294 if not os.path.exists(path):
301 head, tail = ntpath.split(path) # This is for windows?? Recheck
306 head, tail = ntpath.split(path)
310 # Creating Diff between OLD and NEW attribute files v12
311 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
312 if GenerateDiffAttr == "FALSE":
314 with open(ATTR_OLD, 'r') as f_old:
315 lines1 = set(f_old.read().splitlines())
317 with open(ATTR_NEW, 'r') as f_new:
318 lines2 = set(f_new.read().splitlines())
320 lines = set.difference(lines2, lines1)
321 with open(ATTR_FILE, 'w+') as file_out:
323 logging.info('Diff_AttrFiles - %s' % line)
324 file_out.write(line + '\n')
327 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
328 # Full File Path should MATCH
329 if GenerateDiffAttr == "FALSE":
331 FilePath = '"/' + RequestedPath + '"'
332 #print ('FilePath - %s'% (FilePath))
333 with open(AttributeFile) as f:
336 if Type == SYMLINK_TYPE:
337 Sym_Attibutes.append(line)
339 File_Attibutes.append(line)
342 def hash_file(filename):
343 '''This function returns the SHA-1 hash of the file passed into it'''
348 # open file for reading in binary mode
349 with open(filename, 'rb') as file:
350 # loop till the end of the file
353 # read only 1024 bytes at a time
354 chunk = file.read(1024 * 1024)
357 # return the hex representation of digest
361 def find_dupes_dir(BASE_OLD, BASE_NEW):
364 print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
365 logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
366 for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
367 #print('Scanning %s...' % rootbase)
368 for filename in fileListB:
369 path = os.path.join(rootbase, filename)
370 if os.path.islink(path):
373 file_hash = hash_file(path)
374 dups[file_hash] = path
376 for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
377 #print('Scanning %s...' % roottarget)
378 for filename in fileListT:
379 # Get the path to the file
380 path = os.path.join(roottarget, filename)
381 if os.path.islink(path):
384 file_hash = hash_file(path)
385 # Add or append the file path
386 if file_hash in dups:
387 BaseStr = dups.get(file_hash)
388 Baseloc = path.find('/')
389 TarLoc = BaseStr.find('/')
390 if not path[Baseloc:] == BaseStr[TarLoc:]:
391 logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
392 fdupes[path] = BaseStr
393 logging.info('Total Duplicate files %d' % (len(fdupes)))
397 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
400 print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
402 for filename in fileListB:
403 Src_File = BASE_OLD + '/' + filename
404 if os.path.islink(Src_File) or os.path.isdir(Src_File):
407 file_hash = hash_file(Src_File)
408 dups[file_hash] = Src_File
410 for filename in fileListT:
411 Dest_File = BASE_NEW + '/' + filename
412 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
415 file_hash = hash_file(Dest_File)
416 if file_hash in dups:
417 BaseStr = dups.get(file_hash)
418 Baseloc = BaseStr.find('/')
419 if not BaseStr[Baseloc:] == filename:
420 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
421 fdupes[BaseStr] = filename
423 logging.info('Total Duplicate files %d' % (len(fdupes)))
427 def SS_UpdateSize(src_file, dst_file):
430 oldsize_d = os.path.getsize(src_file)
431 newsize_d = os.path.getsize(dst_file)
432 if oldsize_d >= newsize_d:
441 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
442 print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
443 logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
445 # First let's fill up these categories
463 for elt in New_files:
464 if elt not in Old_files:
465 files_new.append(elt)
466 logging.info('New files %s' % elt)
468 # Generate Delete List
469 for elt in Old_files:
470 if elt not in New_files:
471 # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
472 files_removed.append(elt)
473 logging.info('Old files %s' % elt)
476 #print('List of Old Dirs %s' % elt)
477 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
478 if elt not in New_dirs:
479 Dir_removed.append(elt)
480 logging.info('Old Dirs %s' % elt + '/')
483 if elt not in Old_dirs:
484 Dir_Added.append(elt)
485 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
487 # What files have changed contents but not name/path?
488 for elt in New_files:
490 # Both are symbolic linkes and they differ
491 src_file = BASE_OLD + '/' + elt
492 dst_file = BASE_NEW + '/' + elt
493 #print('Files Changed - %s -%s' % (src_file,dst_file))
494 if os.path.islink(src_file) and os.path.islink(dst_file):
495 if not os.readlink(src_file) == os.readlink(dst_file):
496 files_changed.append(elt)
497 #print('%d Sym link files changed' % len(files_changed))
498 logging.info('Sym links Changed - %s' % elt)
500 files_unchanged.append(elt)
501 # Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
502 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
503 if not filecmp.cmp(src_file, dst_file):
504 files_changed.append(elt)
505 #print('%d Normal files changed' % len(files_changed))
506 #print('Files Changed - %s' % elt)
508 files_unchanged.append(elt)
509 # File types differ between BASE and TARGET
511 logging.info('Files are of diff types but same names Src- %s Des- %s' % (src_file, dst_file))
512 # Both file types have changed and they differ
513 # Case 1: First Delete the OLD entry file type (Be it anything)
514 # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
515 files_removed.append(elt)
516 files_new.append(elt)
518 # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
519 # This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
521 print("Check for any verbatim under - %s" % VERBATIM_LIST)
522 if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
523 with open(VERBATIM_LIST, 'r') as F_News:
524 lines = set(F_News.read().splitlines())
526 if line in files_changed:
527 files_changed.remove(line)
528 files_removed.append(line)
529 if line in files_new:
530 files_new.remove(line)
532 # Currently if Version or number is the first character of the file, then we are NOT making any diffs.
533 if SUPPORT_RENAME == "TRUE":
534 for elt in files_removed:
535 if os.path.isfile(BASE_OLD + '/' + elt):
536 FileName = path_leaf(elt)
537 entries = re.split('[0-9]', FileName)
538 # Gives the STRING part of NAME. if name starts with version then later part wil b string
539 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
540 # If version is starting at the begining of the string?? shd we hav additional check for such cases??
541 if len(entries[0]) > 0:
542 files_Del_List.update({entries[0]: elt})
544 for elt in files_new:
545 if os.path.isfile(BASE_NEW + '/' + elt):
546 FileName = path_leaf(elt)
547 entries = re.split('[0-9]', FileName)
548 #print('Entires under NEWfiles list after split - %s %s - %s' % (FileName, entries[0], elt))
549 if len(entries[0]) > 0:
550 files_New_List.update({entries[0]: elt})
552 for key, value in files_Del_List.iteritems():
553 #print('Key value pair -%s -%s' % (key, value))
554 if key in files_New_List:
555 # this file is the same name in both!
556 src_file = BASE_OLD + '/' + value
557 dst_file = BASE_NEW + '/' + files_New_List[key]
558 olddirpath = path_head(files_New_List[key])
559 newdirpath = path_head(value)
560 if os.path.islink(src_file) or os.path.islink(dst_file):
561 logging.debug('Cannot diff as one of them is Symlink')
562 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
563 logging.debug('Cannot diff as one of them is dir')
565 #Pick the best diff of same type and diff names
566 files_renamed.append([files_New_List[key], value])
567 files_removed.remove(value)
568 files_new.remove(files_New_List[key])
572 Partition.txt contains Protocol for UPI
573 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
582 SymLinkDoc = OUT_DIR + '/' + PART_NAME + SYMLINK_DOC_NAME
583 Partition_Doc = open(OUT_DIR + '/' + PART_NAME + '.txt', 'w')
584 Partition_Doc_SymLinks = open(SymLinkDoc, 'w')
586 print("writing diff'ed changed files...")
587 for elt in files_changed:
588 dst_file = BASE_NEW + '/' + elt
589 src_file = BASE_OLD + '/' + elt
590 # Both files are symbolic links and they differ
591 if os.path.islink(dst_file) and os.path.islink(src_file):
592 # Both are symlinks and they differ
593 logging.debug(' File Changed is Link %s ' % dst_file)
594 patch = os.readlink(dst_file)
595 Sym_Diff_Cnt = Sym_Diff_Cnt + 1
596 Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
597 Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
598 # Both are NORMAL files and they differ
599 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
600 # Both are files and they differ
601 Diff_Cnt = Diff_Cnt + 1
602 patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
603 patchLoc = '%s/%s' % (OUT_DIR, patchName)
604 logging.debug(' File Differ %s %s' % (src_file, dst_file))
605 SS_UpdateSize(src_file, dst_file)
608 ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
610 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
611 files_new.append(elt)
612 Diff_Cnt = Diff_Cnt - 1
614 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
616 Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
617 # Both differ but they are of diff types
619 # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
620 files_removed.append(elt)
621 files_new.append(elt)
623 fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
624 for oldpath, newpath in fdupes.iteritems():
625 logging.info('Dupes %s -> %s' % (oldpath, newpath))
627 for elt in files_removed:
628 src_file = BASE_OLD + '/' + elt
629 # If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
630 if src_file in fdupes.keys():
631 dst_file = BASE_NEW + '/' + fdupes[src_file]
632 logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
633 Move_Cnt = Move_Cnt + 1
634 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
635 files_removed.remove(elt)
636 files_new.remove(fdupes[src_file])
638 # Should be placed after removing duplicates, else they will be filtered here.
639 # loop shd b for all NEW files, rather than for all delete files (Current understanding)
640 # First Step: Sort & Filter out unwanted files
641 # Minimum condition used is,
642 # 1. File name should match 70%
643 # 2. Extensions should be same
644 # 3. File name length shd b greater than 3 char
645 # 4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
646 # 5. Should consider editdistance for RENAME LOGIC ==> TBD
648 Base_DelList = files_removed[:]
649 Base_NewList = files_new[:]
650 DelList = sorted(Base_DelList, key=path_leaf)
651 NewList = sorted(Base_NewList, key=path_leaf)
652 logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
656 # Remove unwanted items which we cant make diff with for rename logic
658 if os.path.islink(BASE_OLD + '/' + file):
660 elif os.path.isdir(BASE_OLD + '/' + file):
664 #logging.debug('Sorted del list - %s' % (file))
669 if os.path.islink(BASE_NEW + '/' + file):
671 elif os.path.isdir(BASE_NEW + '/' + file):
673 elif len(path_leaf(file)) <= 3:
674 logging.debug('Ignored for best picks -%s ' % (BASE_NEW + '/' + file))
681 logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
683 for new_file in NewList:
685 DirPathNew = path_head(new_file)
686 FileNameNew = path_leaf(new_file)
688 winning_patch_sz = os.path.getsize(BASE_NEW + '/' + new_file)
689 New_fs = winning_patch_sz
692 for del_file in DelList:
693 FileNameOld = path_leaf(del_file)
694 if (FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
695 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
696 # Percentage difference between two file sizes is within 30%, then we consider for diff generation
697 Del_fs = os.path.getsize(BASE_OLD + '/' + del_file)
698 v1 = abs(New_fs - Del_fs)
699 v2 = (New_fs + Del_fs) / 2
700 if(v2 <= 0 or ((v1 / v2) * 100) > 30):
701 logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
703 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
705 DiffSize = measure_two_filediffs(BASE_OLD + '/' + del_file, BASE_NEW + '/' + new_file)
706 if (DiffSize < 0.8 * winning_patch_sz):
707 winning_patch_sz = DiffSize
708 winning_file = del_file
709 elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and R_Flag == 'TRUE'):
710 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
712 if len(winning_file) > 0:
713 logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
714 files_renamed.append([new_file, winning_file])
715 DelList.remove(winning_file)
716 files_removed.remove(winning_file)
717 files_new.remove(new_file)
719 #********************** Files should NOT be deleted for any such renames ***********************
721 if SUPPORT_RENAME == "TRUE":
722 for elt in files_renamed:
723 src_file = BASE_OLD + '/' + elt[1]
724 dst_file = BASE_NEW + '/' + elt[0]
725 Diff_Cnt = Diff_Cnt + 1
726 patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
727 #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
728 patchLoc = '%s/%s' % (OUT_DIR, patchName)
729 logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
730 # Should be careful of renaming files??
731 # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
732 # What if folder path has numerics??
734 if os.path.isdir(src_file) or os.path.isdir(dst_file):
735 # This case never occurs??
736 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
737 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
738 else: # Make sure these files are PROPER and they shd NOT be symlinks
739 if filecmp.cmp(src_file, dst_file):
740 Move_Cnt = Move_Cnt + 1
741 Diff_Cnt = Diff_Cnt - 1
742 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
745 ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
747 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
748 files_new.append(elt)
749 Diff_Cnt = Diff_Cnt - 1
751 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
753 SS_UpdateSize(src_file, dst_file)
754 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
756 # HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
757 # So that, the rename functionality will NOT create PATCH instead of verbatims.
759 if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
760 with open(VERBATIM_LIST, 'r') as F_News:
761 lines = set(F_News.read().splitlines())
763 if line not in files_new:
764 if os.path.exists(BASE_NEW + '/' + line):
765 files_new.append(line)
766 Verbatim_Cnt = Verbatim_Cnt + 1
767 logging.debug("Added to list of verbatims -%s" % BASE_NEW + '/' + line)
769 for elt in files_removed:
770 # if files are part of patches after renaming, we shd remove them as part of removed.
771 src_file = BASE_OLD + '/' + elt
772 if os.path.islink(src_file):
773 Partition_Doc.write('DEL:SYM:%s\n' % (elt))
774 elif os.path.isdir(src_file):
775 # If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
776 Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
778 Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
779 logging.debug(' File Deleted %s' % src_file)
780 Del_Cnt = Del_Cnt + 1
782 Dir_removed.sort(reverse=True)
783 for elt in Dir_removed:
784 # if Dir is empty, add it to the removed list.
785 src_file = BASE_OLD + '/' + elt
786 # Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
787 # It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
788 if os.path.isdir(src_file):
789 Partition_Doc.write('DEL:END:%s\n' % (elt))
790 Del_Cnt = Del_Cnt + 1
791 logging.debug(' Dir Deleted- %s' % src_file)
793 for elt in files_new:
794 dst_file = BASE_NEW + '/' + elt
795 newfiles_dest_path = 'run/upgrade-sysroot/'
796 ensure_dir_exists(newfiles_dest_path)
797 if os.path.islink(dst_file):
798 patch = os.readlink(dst_file)
799 logging.debug(' File New Links %s' % elt)
800 Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
801 # What if this is only a new sym link and folder already exists??? Should recheck
802 destpath = newfiles_dest_path + elt
803 if not os.path.exists(path_head(destpath)):
804 os.makedirs(path_head(destpath))
805 logging.info('New SymLink - Adding missing Dir')
806 #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
807 Sym_New_Cnt = Sym_New_Cnt + 1
808 elif os.path.isdir(dst_file): # We create just empty directory here
809 destpath = newfiles_dest_path + elt
810 if not os.path.exists(destpath):
811 os.makedirs(destpath)
812 logging.debug(' File New Dir %s' % destpath)
813 New_Cnt = New_Cnt + 1
815 New_Cnt = New_Cnt + 1
816 destpath = newfiles_dest_path + elt
817 destdir = os.path.dirname(destpath)
818 logging.debug('New files - %s ==> %s' % (dst_file, destdir))
820 if not os.path.isdir(destdir):
823 except Exception as exc:
824 logging.critical('Error in NEW files DIR entry -%s' % destdir)
828 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
829 shutil.copy2(dst_file, destpath)
830 logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
831 except Exception as exc:
832 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
835 for elt in Dir_Added:
836 newfiles_dest_path = 'run/upgrade-sysroot/'
837 ensure_dir_exists(newfiles_dest_path)
838 destpath = newfiles_dest_path + elt
839 if not os.path.exists(destpath):
840 os.makedirs(destpath)
841 logging.debug(' DirList New Dir %s' % destpath)
842 New_Cnt = New_Cnt + 1
844 # Base directory should be system
845 print 'Compressing New files'
846 if (New_Cnt > 0 or Sym_New_Cnt > 0):
847 WorkingDir = os.getcwd()
848 os.chdir(os.getcwd() + "/" + NEW_FILES_PATH)
849 logging.info('Curr Working Dir - %s' % os.getcwd())
850 os.system(ZIPUTIL + NEW_FILES_PATH + " >> " + LOGFILE)
851 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR)
852 # New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
853 SS_UpdateSize(WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME)
855 shutil.rmtree(NEW_FILES_PATH)
856 # use 7z a system.7z ./*
858 #logging.info('%d Dir to be removed' % len(Dir_removed))
859 logging.info('%d files unchanged' % len(files_unchanged))
860 logging.info('%d files files_renamed' % len(files_renamed))
861 logging.info('%d files NEW' % len(files_new))
862 logging.info('%d File attr' % len(File_Attibutes))
863 logging.info('%d Sym attr' % len(Sym_Attibutes))
864 logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
865 print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
867 # There could be duplicates, TODO, can check before adding..
868 ATTR_FILE_D = open(ATTR_FILE, 'a+')
869 for elt in File_Attibutes:
870 ATTR_FILE_D.write(elt)
871 for elt in Sym_Attibutes:
872 ATTR_FILE_D.write(elt)
876 Partition_Doc_SymLinks.close()
877 Partition_Read_SymLinks = open(SymLinkDoc, 'r+')
878 Partition_Doc.write(Partition_Read_SymLinks.read())
879 Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
880 Partition_Doc_SymLinks.close()
881 Partition_Doc.close()
882 os.remove(SymLinkDoc)
884 if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
885 print('No Delta Generated for %s - %s' % (PART_NAME, OUT_DIR))
886 logging.info('No Delta Generated for %s' % PART_NAME)
887 shutil.rmtree(OUT_DIR)
890 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
892 # CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
893 print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
894 shutil.copy2(g_output_dir + '/' + b_apk, g_output_dir + '/temp')
895 temp_apk = '../' + g_output_dir + '/' + b_apk
896 Patch = 'Patch_' + b_apk
897 ensure_dir_exists(Patch)
898 shutil.copy2(g_output_dir + '/' + b_apk, Patch + '/' + b_apk)
900 # Size issue on Device side?? shd check this
901 subprocess.call(['unzip', '-q', Patch + '/' + b_apk, '-d', Patch])
902 with open(g_output_dir + '/PATCH.txt', 'r') as f_new:
903 lines = set(f_new.read().splitlines())
905 #print('Action ==> %s' % line)
906 #Action, Path, Patch = line.split('|')
907 Items = line.split('|')
910 ActualPath = a_folder + '/' + Path
911 PatchPath = Patch + '/' + Path
912 SrcPath = g_output_dir + '/' + path_leaf(Path)
913 #print('Action ==> %s Path ==> %s ' % (Action, Path))
915 patchName = g_output_dir + '/' + Items[2]
916 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
917 subprocess.call([DIFFPATCH_UTIL, ActualPath, ActualPath, patchName])
918 WorkingDir = os.getcwd()
919 os.chdir(WorkingDir + "/" + "temp_a")
920 subprocess.call(['cp', '--parents', Path, '../' + Patch])
923 WorkingDir = os.getcwd()
924 os.chdir(WorkingDir + "/" + "temp_a")
925 subprocess.call(['cp', '--parents', Path, '../' + Patch])
928 print('Apply_Container_Delta - Unknown Error')
929 #print('Touch all files and set common attributes for DIFF generation')
930 WorkingDir = os.getcwd()
931 os.chdir(WorkingDir + "/" + Patch)
933 CONTAINER_DATE = '200011111111.11'
934 CONTAINER_MODE = '0755'
935 subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
936 subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
937 subprocess.call(['chmod', '-R', CONTAINER_MODE, '../' + Patch])
939 print 'Update Intermediate Archive'
940 #subprocess.call(['zip','-ryX', b_apk, '*'])
941 subprocess.call(['zip', '-ryX', b_apk] + glob.glob('*'))
943 #print('Apply Path completed - Now create diff for this and place in patch folder')
945 print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir + '/' + b_apk, new_apk))
946 patchName = ('New' + '_%s' + DIFF_SUFFIX) % (b_apk)
947 patchLoc = '%s/%s' % (g_output_dir, patchName)
949 subprocess.call([DIFF_UTIL, Patch + '/' + b_apk, new_apk, patchLoc])
951 # Only on HOST... for testing
952 if TEST_MODE == 'TRUE':
953 UpgradedName = '%s_Upgraded' % (b_apk)
954 subprocess.call([DIFFPATCH_UTIL, Patch + '/' + b_apk, UpgradedName, patchLoc])
956 # This is file only with NEWS and empty diffs and same files.
957 if TEST_MODE == 'FALSE':
958 os.remove(g_output_dir + '/' + b_apk)
959 os.rename(g_output_dir + '/temp', g_output_dir + '/' + b_apk)
964 return (info.external_attr >> 16) == 0120777
967 def NewFiles(src, dest):
969 subprocess.call(['cp', '-rp', src, dest])
971 #shutil.copytree(src, dest)
972 #except OSError as e:
973 # If the error was caused because the source wasn't a directory
974 #if e.errno == errno.ENOTDIR:
975 #shutil.copy2(src, dest)
977 #print('Directory not copied. Error: %s' % e)
980 def measure_two_filediffs(src, dst):
981 patchLoc = 'temp.patch'
982 subprocess.call([DIFF_UTIL, src, dst, patchLoc])
983 result_size = os.path.getsize(patchLoc)
992 for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
993 for directory in directories:
994 #DirName = os.path.join(root+'/',directory)
995 DirName = os.path.join(root, directory)
996 if os.path.islink(DirName):
997 logging.debug('This is symlink pointing to dir -%s' % DirName)
998 all_files.append(os.path.relpath(DirName, path))
999 elif not os.listdir(DirName):
1000 #print('*****Empty Directory******* -%s', DirName)
1001 # This should NOT be appended ??? Empty dir shd b considered
1002 all_dirs.append(os.path.relpath(DirName, path))
1004 all_dirs.append(os.path.relpath(DirName, path))
1005 for filename in filenames:
1006 FileName = os.path.join(root, filename)
1007 all_files.append(os.path.relpath(FileName, path))
1011 return all_files, all_dirs
1014 USAGE_DOCSTRING = """
1015 Generate Delta using BASEOLD AND BASE NEW
1016 Attributes is optional
1017 Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1021 def Usage(docstring):
1022 print docstring.rstrip("\n")
1023 print COMMON_DOCSTRING
1026 if __name__ == '__main__':