21 if sys.hexversion < 0x02040000:
22 print >> sys.stderr, "Python 2.4 or newer is required."
26 Diff two folders and create delta using SS_BSDIFF
27 Will maintain same format of script that will be generated when we use diffutil
29 1. Create a list of files in each Base folders,
30 2. These files will fall into one these below categories:
31 1) Only in OLD - Should be deleted
32 2) Only in NEW - Should be added or renamed accordingly
33 3) File exists in both directories but contents are different - Create Diff.
34 4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
35 5) Duplicates in the list of Deletes and News
36 6) Close matching diffs even though name changes across directories. (for matching extension)
37 7) Clearing empty directories after Moves or diffs under Rename.
38 8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
41 1. Given two folders, from list of REMOVED and NEW files find if there
42 is version change and create diff between them
45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
46 Catching errors at all stages. SHOULD exit & return error in case of failure
54 global NEW_FILES_ZIP_NAME
57 global SYMLINK_DOC_NAME
63 global SUPPORT_CONTAINERS
68 global COMMON_BIN_PATH
75 COMMON_BIN_PATH = "../../common/bin/"
76 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
77 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
79 ZIPUTIL = "7z -mf=off a system.7z "
80 NEW_FILES_PATH = "run/upgrade-sysroot"
81 NEW_FILES_ZIP_NAME = "system.7z"
83 ATTR_DOC_EXT = "_attr.txt"
84 SYMLINK_DOC_NAME = "_sym.txt"
87 DIFF_SUFFIX = ".delta"
89 FULL_IMAGE = "FULL_IMAGE"
90 DELTA_IMAGE = "DELTA_IMAGE"
94 VERBATIM_LIST = "Verbatim_List.txt"
98 COMPRESSION_LZMA = "lzma"
99 COMPRESSION_BROTLI = "brotli"
101 SUPPORT_RENAME = "TRUE" # Use appropriate name
102 SUPPORT_CONTAINERS = "FALSE"
103 SUPPORT_VERBATIM = "TRUE"
109 logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
111 global GenerateDiffAttr
114 if len(sys.argv) < 5:
115 sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
116 UPDATE_TYPE = sys.argv[1]
117 UPDATE_TYPE_S = UPDATE_TYPE.split(":")
118 PART_NAME = sys.argv[2] # lets make this also optional
120 BASE_OLD = sys.argv[3]
121 BASE_NEW = sys.argv[4]
122 OUT_DIR = sys.argv[5]
125 UPDATE_CFG_PATH = EMPTY
126 GenerateDiffAttr = "FALSE"
127 if UPDATE_TYPE_S[0] == DELTA_FS:
128 #instead of arguments check it in outdirectory ?
129 if len(sys.argv) == 9:
130 ATTR_OLD = sys.argv[6]
131 ATTR_NEW = sys.argv[7]
132 UPDATE_CFG_PATH = '../' + sys.argv[8]
133 GenerateDiffAttr = "TRUE"
135 elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]:
136 if len(sys.argv) == 7:
137 #Use path in better way
138 UPDATE_CFG_PATH = '../' + sys.argv[6]
141 global DIFFPATCH_UTIL
142 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
143 DIFF_UTIL = COMMON_BIN_PATH + DIFF_UTIL
144 DIFFPATCH_UTIL = COMMON_BIN_PATH + DIFFPATCH_UTIL
145 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
146 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
147 logging.info('Diff Util Does NOT exist -- ABORT')
150 start = datetime.datetime.now().time()
151 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
152 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s] [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]' % (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
155 ensure_dir_exists(OUT_DIR)
156 except FileExistsError as exc:
157 logging.error('Argument passed as OUT_DIR - %s is already an existing file' % OUT_DIR)
159 if GenerateDiffAttr == "TRUE":
160 if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
161 print >> sys.stderr, "Attributes missing -- ABORT"
164 # Should check if APT is supported on other linux flavours
166 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
167 logging.info('Basic utils installed')
169 print >> sys.stderr, "Basic utils missing -- ABORT"
172 if UPDATE_TYPE_S[0] == FULL_IMAGE:
173 SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
174 # #### currently does not support LZMA ####
175 # elif UPDATE_TYPE == DELTA_IMAGE:
176 # SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
177 elif UPDATE_TYPE_S[0] == DELTA_IMAGE:
178 SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
179 elif UPDATE_TYPE == DELTA_FS:
180 AttributeFile = ATTR_NEW
181 ATTR_FILE = OUT_DIR + '/' + PART_NAME + ATTR_DOC_EXT
182 Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
183 Old_files, Old_dirs = Get_Files(BASE_OLD)
184 New_files, New_dirs = Get_Files(BASE_NEW)
185 SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
187 if not UPDATE_CFG_PATH == EMPTY:
188 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
190 elif UPDATE_TYPE == EXTRA:
191 print('UPDATE_TYPE ---- EXTRA')
193 print('UPDATE_TYPE ---- UNKNOWN FORMAT')
195 if GenerateDiffAttr == "TRUE":
196 if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
199 end = datetime.datetime.now().time()
201 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
202 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
203 logging.info('Time start [%s] - Time end [%s]' % (start, end))
204 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
206 except Exception as exc:
207 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
211 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
212 f = open(UPDATE_CFG_PATH, 'r')
213 lines = f.readlines()
215 f = open(UPDATE_CFG_PATH, 'w')
217 ConfigItems = line.split()
218 if ConfigItems[0] == DELTA_BIN:
219 DELTA = ConfigItems[1]
220 logging.info('Updating %s config' % DELTA_BIN)
221 line = line.rstrip('\n')
223 line = line.replace(line, line + '\t' + str(Value) + '\n')
230 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
233 oldsize_d = os.path.getsize(BASE_OLD)
234 newsize_d = os.path.getsize(BASE_NEW)
235 SHA_BIN_DEST = hash_file(BASE_NEW)
236 SHA_BIN_BASE = hash_file(BASE_OLD)
238 #incase UPDATE CFG is empty
240 SS_UpdateSize(BASE_OLD, BASE_NEW)
241 #Should throw error if PART NAME NOT found??
242 if not UPDATE_CFG_PATH == EMPTY:
243 f = open(UPDATE_CFG_PATH, 'r')
244 lines = f.readlines()
246 f = open(UPDATE_CFG_PATH, 'w')
248 ConfigItems = line.split()
249 if ConfigItems[0] == DELTA_BIN:
250 logging.info('Updating %s config' % DELTA_BIN)
251 DELTA = ConfigItems[1]
252 line = line.rstrip('\n')
253 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
259 patchLoc = '%s/%s' % (OUT_DIR, DELTA)
260 logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW, DELTA_BIN, patchLoc))
261 subprocess.call([DIFF_UTIL, "-c", COMPRESSION_METHOD, BASE_OLD, BASE_NEW, patchLoc])
264 def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
265 logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW, DELTA_BIN))
266 oldsize_d = os.path.getsize(BASE_OLD)
267 newsize_d = os.path.getsize(BASE_NEW)
268 SHA_BIN_DEST = hash_file(BASE_NEW)
269 SHA_BIN_BASE = hash_file(BASE_OLD)
270 #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
271 SS_UpdateSize(BASE_OLD, BASE_NEW)
273 if not UPDATE_CFG_PATH == EMPTY:
274 f = open(UPDATE_CFG_PATH, 'r')
275 lines = f.readlines()
277 f = open(UPDATE_CFG_PATH, 'w')
279 ConfigItems = line.split()
280 if ConfigItems[0] == DELTA_BIN:
281 logging.info('Updating %s config' % DELTA_BIN)
282 DELTA = ConfigItems[1]
283 line = line.rstrip('\n')
284 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
291 def zipdir(path, zip):
292 for root, dirs, files in os.walk(path):
294 zip.write(os.path.join(root, file))
297 def ensure_dir_exists(path):
298 if not os.path.exists(path):
300 elif os.path.isfile(path):
301 raise FileExistsError
307 head, tail = ntpath.split(path) # This is for windows?? Recheck
312 head, tail = ntpath.split(path)
316 # Creating Diff between OLD and NEW attribute files v12
317 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
318 if GenerateDiffAttr == "FALSE":
320 with open(ATTR_OLD, 'r') as f_old:
321 lines1 = set(f_old.read().splitlines())
323 with open(ATTR_NEW, 'r') as f_new:
324 lines2 = set(f_new.read().splitlines())
326 lines = set.difference(lines2, lines1)
327 with open(ATTR_FILE, 'w+') as file_out:
329 logging.info('Diff_AttrFiles - %s' % line)
330 file_out.write(line + '\n')
333 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
334 # Full File Path should MATCH
335 if GenerateDiffAttr == "FALSE":
337 FilePath = '"/' + RequestedPath + '"'
338 #print ('FilePath - %s'% (FilePath))
339 with open(AttributeFile) as f:
342 if Type == SYMLINK_TYPE:
343 Sym_Attibutes.append(line)
345 File_Attibutes.append(line)
348 def hash_file(filename):
349 '''This function returns the SHA-1 hash of the file passed into it'''
354 # open file for reading in binary mode
355 with open(filename, 'rb') as file:
356 # loop till the end of the file
359 # read only 1024 bytes at a time
360 chunk = file.read(1024 * 1024)
363 # return the hex representation of digest
367 def find_dupes_dir(BASE_OLD, BASE_NEW):
370 print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
371 logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
372 for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
373 #print('Scanning %s...' % rootbase)
374 for filename in fileListB:
375 path = os.path.join(rootbase, filename)
376 if os.path.islink(path):
379 file_hash = hash_file(path)
380 dups[file_hash] = path
382 for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
383 #print('Scanning %s...' % roottarget)
384 for filename in fileListT:
385 # Get the path to the file
386 path = os.path.join(roottarget, filename)
387 if os.path.islink(path):
390 file_hash = hash_file(path)
391 # Add or append the file path
392 if file_hash in dups:
393 BaseStr = dups.get(file_hash)
394 Baseloc = path.find('/')
395 TarLoc = BaseStr.find('/')
396 if not path[Baseloc:] == BaseStr[TarLoc:]:
397 logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
398 fdupes[path] = BaseStr
399 logging.info('Total Duplicate files %d' % (len(fdupes)))
403 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
406 print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
408 for filename in fileListB:
409 Src_File = BASE_OLD + '/' + filename
410 if os.path.islink(Src_File) or os.path.isdir(Src_File):
413 file_hash = hash_file(Src_File)
414 dups[file_hash] = Src_File
416 for filename in fileListT:
417 Dest_File = BASE_NEW + '/' + filename
418 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
421 file_hash = hash_file(Dest_File)
422 if file_hash in dups:
423 BaseStr = dups.get(file_hash)
424 Baseloc = BaseStr.find('/')
425 if not BaseStr[Baseloc:] == filename:
426 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
427 fdupes[BaseStr] = filename
429 logging.info('Total Duplicate files %d' % (len(fdupes)))
433 def SS_UpdateSize(src_file, dst_file):
436 oldsize_d = os.path.getsize(src_file)
437 newsize_d = os.path.getsize(dst_file)
438 if oldsize_d >= newsize_d:
447 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
448 print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
449 logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
451 # First let's fill up these categories
469 for elt in New_files:
470 if elt not in Old_files:
471 files_new.append(elt)
472 logging.info('New files %s' % elt)
474 # Generate Delete List
475 for elt in Old_files:
476 if elt not in New_files:
477 # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
478 files_removed.append(elt)
479 logging.info('Old files %s' % elt)
482 #print('List of Old Dirs %s' % elt)
483 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
484 if elt not in New_dirs:
485 Dir_removed.append(elt)
486 logging.info('Old Dirs %s' % elt + '/')
489 if elt not in Old_dirs:
490 Dir_Added.append(elt)
491 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
493 # What files have changed contents but not name/path?
494 for elt in New_files:
496 # Both are symbolic linkes and they differ
497 src_file = BASE_OLD + '/' + elt
498 dst_file = BASE_NEW + '/' + elt
499 #print('Files Changed - %s -%s' % (src_file,dst_file))
500 if os.path.islink(src_file) and os.path.islink(dst_file):
501 if not os.readlink(src_file) == os.readlink(dst_file):
502 files_changed.append(elt)
503 #print('%d Sym link files changed' % len(files_changed))
504 logging.info('Sym links Changed - %s' % elt)
506 files_unchanged.append(elt)
507 # Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
508 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
509 if not filecmp.cmp(src_file, dst_file):
510 files_changed.append(elt)
511 #print('%d Normal files changed' % len(files_changed))
512 #print('Files Changed - %s' % elt)
514 files_unchanged.append(elt)
515 # File types differ between BASE and TARGET
517 logging.info('Files are of diff types but same names Src- %s Des- %s' % (src_file, dst_file))
518 # Both file types have changed and they differ
519 # Case 1: First Delete the OLD entry file type (Be it anything)
520 # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
521 files_removed.append(elt)
522 files_new.append(elt)
524 # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
525 # This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
527 print("Check for any verbatim under - %s" % VERBATIM_LIST)
528 if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
529 with open(VERBATIM_LIST, 'r') as F_News:
530 lines = set(F_News.read().splitlines())
532 if line in files_changed:
533 files_changed.remove(line)
534 files_removed.append(line)
535 if line in files_new:
536 files_new.remove(line)
538 # Currently if Version or number is the first character of the file, then we are NOT making any diffs.
539 if SUPPORT_RENAME == "TRUE":
540 for elt in files_removed:
541 if os.path.isfile(BASE_OLD + '/' + elt):
542 FileName = path_leaf(elt)
543 entries = re.split('[0-9]', FileName)
544 # Gives the STRING part of NAME. if name starts with version then later part wil b string
545 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
546 # If version is starting at the begining of the string?? shd we hav additional check for such cases??
547 if len(entries[0]) > 0:
548 files_Del_List.update({entries[0]: elt})
550 for elt in files_new:
551 if os.path.isfile(BASE_NEW + '/' + elt):
552 FileName = path_leaf(elt)
553 entries = re.split('[0-9]', FileName)
554 #print('Entires under NEWfiles list after split - %s %s - %s' % (FileName, entries[0], elt))
555 if len(entries[0]) > 0:
556 files_New_List.update({entries[0]: elt})
558 for key, value in files_Del_List.iteritems():
559 #print('Key value pair -%s -%s' % (key, value))
560 if key in files_New_List:
561 # this file is the same name in both!
562 src_file = BASE_OLD + '/' + value
563 dst_file = BASE_NEW + '/' + files_New_List[key]
564 olddirpath = path_head(files_New_List[key])
565 newdirpath = path_head(value)
566 if os.path.islink(src_file) or os.path.islink(dst_file):
567 logging.debug('Cannot diff as one of them is Symlink')
568 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
569 logging.debug('Cannot diff as one of them is dir')
571 #Pick the best diff of same type and diff names
572 files_renamed.append([files_New_List[key], value])
573 files_removed.remove(value)
574 files_new.remove(files_New_List[key])
578 Partition.txt contains Protocol for UPI
579 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
588 SymLinkDoc = OUT_DIR + '/' + PART_NAME + SYMLINK_DOC_NAME
589 Partition_Doc = open(OUT_DIR + '/' + PART_NAME + '.txt', 'w')
590 Partition_Doc_SymLinks = open(SymLinkDoc, 'w')
592 print("writing diff'ed changed files...")
593 for elt in files_changed:
594 dst_file = BASE_NEW + '/' + elt
595 src_file = BASE_OLD + '/' + elt
596 # Both files are symbolic links and they differ
597 if os.path.islink(dst_file) and os.path.islink(src_file):
598 # Both are symlinks and they differ
599 logging.debug(' File Changed is Link %s ' % dst_file)
600 patch = os.readlink(dst_file)
601 Sym_Diff_Cnt = Sym_Diff_Cnt + 1
602 Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
603 Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
604 # Both are NORMAL files and they differ
605 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
606 # Both are files and they differ
607 Diff_Cnt = Diff_Cnt + 1
608 patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
609 patchLoc = '%s/%s' % (OUT_DIR, patchName)
610 logging.debug(' File Differ %s %s' % (src_file, dst_file))
611 SS_UpdateSize(src_file, dst_file)
614 ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
616 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
617 files_new.append(elt)
618 Diff_Cnt = Diff_Cnt - 1
620 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
622 Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
623 # Both differ but they are of diff types
625 # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
626 files_removed.append(elt)
627 files_new.append(elt)
629 fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
630 for oldpath, newpath in fdupes.iteritems():
631 logging.info('Dupes %s -> %s' % (oldpath, newpath))
633 for elt in files_removed:
634 src_file = BASE_OLD + '/' + elt
635 # If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
636 if src_file in fdupes.keys():
637 dst_file = BASE_NEW + '/' + fdupes[src_file]
638 logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
639 Move_Cnt = Move_Cnt + 1
640 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
641 files_removed.remove(elt)
642 files_new.remove(fdupes[src_file])
644 # Should be placed after removing duplicates, else they will be filtered here.
645 # loop shd b for all NEW files, rather than for all delete files (Current understanding)
646 # First Step: Sort & Filter out unwanted files
647 # Minimum condition used is,
648 # 1. File name should match 70%
649 # 2. Extensions should be same
650 # 3. File name length shd b greater than 3 char
651 # 4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
652 # 5. Should consider editdistance for RENAME LOGIC ==> TBD
654 Base_DelList = files_removed[:]
655 Base_NewList = files_new[:]
656 DelList = sorted(Base_DelList, key=path_leaf)
657 NewList = sorted(Base_NewList, key=path_leaf)
658 logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
662 # Remove unwanted items which we cant make diff with for rename logic
664 if os.path.islink(BASE_OLD + '/' + file):
666 elif os.path.isdir(BASE_OLD + '/' + file):
670 #logging.debug('Sorted del list - %s' % (file))
675 if os.path.islink(BASE_NEW + '/' + file):
677 elif os.path.isdir(BASE_NEW + '/' + file):
679 elif len(path_leaf(file)) <= 3:
680 logging.debug('Ignored for best picks -%s ' % (BASE_NEW + '/' + file))
687 logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
689 for new_file in NewList:
691 DirPathNew = path_head(new_file)
692 FileNameNew = path_leaf(new_file)
694 winning_patch_sz = os.path.getsize(BASE_NEW + '/' + new_file)
695 New_fs = winning_patch_sz
698 for del_file in DelList:
699 FileNameOld = path_leaf(del_file)
700 if (FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
701 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
702 # Percentage difference between two file sizes is within 30%, then we consider for diff generation
703 Del_fs = os.path.getsize(BASE_OLD + '/' + del_file)
704 v1 = abs(New_fs - Del_fs)
705 v2 = (New_fs + Del_fs) / 2
706 if(v2 <= 0 or ((v1 / v2) * 100) > 30):
707 logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
709 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
711 DiffSize = measure_two_filediffs(BASE_OLD + '/' + del_file, BASE_NEW + '/' + new_file)
712 if (DiffSize < 0.8 * winning_patch_sz):
713 winning_patch_sz = DiffSize
714 winning_file = del_file
715 elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and R_Flag == 'TRUE'):
716 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
718 if len(winning_file) > 0:
719 logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
720 files_renamed.append([new_file, winning_file])
721 DelList.remove(winning_file)
722 files_removed.remove(winning_file)
723 files_new.remove(new_file)
725 #********************** Files should NOT be deleted for any such renames ***********************
727 if SUPPORT_RENAME == "TRUE":
728 for elt in files_renamed:
729 src_file = BASE_OLD + '/' + elt[1]
730 dst_file = BASE_NEW + '/' + elt[0]
731 Diff_Cnt = Diff_Cnt + 1
732 patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
733 #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
734 patchLoc = '%s/%s' % (OUT_DIR, patchName)
735 logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
736 # Should be careful of renaming files??
737 # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
738 # What if folder path has numerics??
740 if os.path.isdir(src_file) or os.path.isdir(dst_file):
741 # This case never occurs??
742 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
743 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
744 else: # Make sure these files are PROPER and they shd NOT be symlinks
745 if filecmp.cmp(src_file, dst_file):
746 Move_Cnt = Move_Cnt + 1
747 Diff_Cnt = Diff_Cnt - 1
748 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
751 ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
753 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
754 files_new.append(elt)
755 Diff_Cnt = Diff_Cnt - 1
757 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
759 SS_UpdateSize(src_file, dst_file)
760 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
762 # HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
763 # So that, the rename functionality will NOT create PATCH instead of verbatims.
765 if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
766 with open(VERBATIM_LIST, 'r') as F_News:
767 lines = set(F_News.read().splitlines())
769 if line not in files_new:
770 if os.path.exists(BASE_NEW + '/' + line):
771 files_new.append(line)
772 Verbatim_Cnt = Verbatim_Cnt + 1
773 logging.debug("Added to list of verbatims -%s" % BASE_NEW + '/' + line)
775 for elt in files_removed:
776 # if files are part of patches after renaming, we shd remove them as part of removed.
777 src_file = BASE_OLD + '/' + elt
778 if os.path.islink(src_file):
779 Partition_Doc.write('DEL:SYM:%s\n' % (elt))
780 elif os.path.isdir(src_file):
781 # If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
782 Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
784 Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
785 logging.debug(' File Deleted %s' % src_file)
786 Del_Cnt = Del_Cnt + 1
788 Dir_removed.sort(reverse=True)
789 for elt in Dir_removed:
790 # if Dir is empty, add it to the removed list.
791 src_file = BASE_OLD + '/' + elt
792 # Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
793 # It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
794 if os.path.isdir(src_file):
795 Partition_Doc.write('DEL:END:%s\n' % (elt))
796 Del_Cnt = Del_Cnt + 1
797 logging.debug(' Dir Deleted- %s' % src_file)
799 for elt in files_new:
800 dst_file = BASE_NEW + '/' + elt
801 newfiles_dest_path = 'run/upgrade-sysroot/'
803 ensure_dir_exists(newfiles_dest_path)
804 except FileExistsError as exc:
805 logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path)
807 if os.path.islink(dst_file):
808 patch = os.readlink(dst_file)
809 logging.debug(' File New Links %s' % elt)
810 Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
811 # What if this is only a new sym link and folder already exists??? Should recheck
812 destpath = newfiles_dest_path + elt
813 if not os.path.exists(path_head(destpath)):
814 os.makedirs(path_head(destpath))
815 logging.info('New SymLink - Adding missing Dir')
816 #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
817 Sym_New_Cnt = Sym_New_Cnt + 1
818 elif os.path.isdir(dst_file): # We create just empty directory here
819 destpath = newfiles_dest_path + elt
820 if not os.path.exists(destpath):
821 os.makedirs(destpath)
822 logging.debug(' File New Dir %s' % destpath)
823 New_Cnt = New_Cnt + 1
825 New_Cnt = New_Cnt + 1
826 destpath = newfiles_dest_path + elt
827 destdir = os.path.dirname(destpath)
828 logging.debug('New files - %s ==> %s' % (dst_file, destdir))
830 if not os.path.isdir(destdir):
833 except Exception as exc:
834 logging.critical('Error in NEW files DIR entry -%s' % destdir)
838 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
839 shutil.copy2(dst_file, destpath)
840 logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
841 except Exception as exc:
842 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
845 for elt in Dir_Added:
846 newfiles_dest_path = 'run/upgrade-sysroot/'
848 ensure_dir_exists(newfiles_dest_path)
849 except FileExistsError as exc:
850 logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path)
852 destpath = newfiles_dest_path + elt
853 if not os.path.exists(destpath):
854 os.makedirs(destpath)
855 logging.debug(' DirList New Dir %s' % destpath)
856 New_Cnt = New_Cnt + 1
858 # Base directory should be system
859 print 'Compressing New files'
860 if (New_Cnt > 0 or Sym_New_Cnt > 0):
861 WorkingDir = os.getcwd()
862 os.chdir(os.getcwd() + "/" + NEW_FILES_PATH)
863 logging.info('Curr Working Dir - %s' % os.getcwd())
864 os.system(ZIPUTIL + NEW_FILES_PATH + " >> " + LOGFILE)
865 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR)
866 # New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
867 SS_UpdateSize(WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME)
869 shutil.rmtree(NEW_FILES_PATH)
870 # use 7z a system.7z ./*
872 #logging.info('%d Dir to be removed' % len(Dir_removed))
873 logging.info('%d files unchanged' % len(files_unchanged))
874 logging.info('%d files files_renamed' % len(files_renamed))
875 logging.info('%d files NEW' % len(files_new))
876 logging.info('%d File attr' % len(File_Attibutes))
877 logging.info('%d Sym attr' % len(Sym_Attibutes))
878 logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
879 print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
881 # There could be duplicates, TODO, can check before adding..
882 ATTR_FILE_D = open(ATTR_FILE, 'a+')
883 for elt in File_Attibutes:
884 ATTR_FILE_D.write(elt)
885 for elt in Sym_Attibutes:
886 ATTR_FILE_D.write(elt)
890 Partition_Doc_SymLinks.close()
891 Partition_Read_SymLinks = open(SymLinkDoc, 'r+')
892 Partition_Doc.write(Partition_Read_SymLinks.read())
893 Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
894 Partition_Doc_SymLinks.close()
895 Partition_Doc.close()
896 os.remove(SymLinkDoc)
898 if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
899 print('No Delta Generated for %s - %s' % (PART_NAME, OUT_DIR))
900 logging.info('No Delta Generated for %s' % PART_NAME)
901 shutil.rmtree(OUT_DIR)
905 return (info.external_attr >> 16) == 0120777
908 def NewFiles(src, dest):
910 subprocess.call(['cp', '-rp', src, dest])
912 #shutil.copytree(src, dest)
913 #except OSError as e:
914 # If the error was caused because the source wasn't a directory
915 #if e.errno == errno.ENOTDIR:
916 #shutil.copy2(src, dest)
918 #print('Directory not copied. Error: %s' % e)
921 def measure_two_filediffs(src, dst):
922 patchLoc = 'temp.patch'
923 subprocess.call([DIFF_UTIL, src, dst, patchLoc])
924 result_size = os.path.getsize(patchLoc)
933 for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
934 for directory in directories:
935 #DirName = os.path.join(root+'/',directory)
936 DirName = os.path.join(root, directory)
937 if os.path.islink(DirName):
938 logging.debug('This is symlink pointing to dir -%s' % DirName)
939 all_files.append(os.path.relpath(DirName, path))
940 elif not os.listdir(DirName):
941 #print('*****Empty Directory******* -%s', DirName)
942 # This should NOT be appended ??? Empty dir shd b considered
943 all_dirs.append(os.path.relpath(DirName, path))
945 all_dirs.append(os.path.relpath(DirName, path))
946 for filename in filenames:
947 FileName = os.path.join(root, filename)
948 all_files.append(os.path.relpath(FileName, path))
952 return all_files, all_dirs
955 USAGE_DOCSTRING = """
956 Generate Delta using BASEOLD AND BASE NEW
957 Attributes is optional
958 Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
962 def Usage(docstring):
963 print docstring.rstrip("\n")
964 print COMMON_DOCSTRING
967 if __name__ == '__main__':