21 if sys.hexversion < 0x02040000:
22 print >> sys.stderr, "Python 2.4 or newer is required."
26 Diff two folders and create delta using SS_BSDIFF
27 Will maintain same format of script that will be generated when we use diffutil
29 1. Create a list of files in each Base folders,
30 2. These files will fall into one these below categories:
31 1) Only in OLD - Should be deleted
32 2) Only in NEW - Should be added or renamed accordingly
33 3) File exists in both directories but contents are different - Create Diff.
34 4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
35 5) Duplicates in the list of Deletes and News
36 6) Close matching diffs even though name changes across directories. (for matching extension)
37 7) Clearing empty directories after Moves or diffs under Rename.
38 8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
41 1. Given two folders, from list of REMOVED and NEW files find if there
42 is version change and create diff between them
45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
46 Catching errors at all stages. SHOULD exit & return error in case of failure
53 global NEW_FILES_FOLDER
54 global NEW_FILES_ZIP_NAME
57 global SYMLINK_DOC_NAME
63 global SUPPORT_CONTAINERS
68 global COMMON_BIN_PATH
74 COMMON_BIN_PATH = "../../common/bin/"
75 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
76 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
78 #ZIPUTIL = "7z a system.7z "
79 NEW_FILES_PATH = "system"
80 NEW_FILES_FOLDER = "system"
81 NEW_FILES_ZIP_NAME = "system.7z"
83 ATTR_DOC_EXT = "_attr.txt"
84 SYMLINK_DOC_NAME = "_sym.txt"
87 DIFF_SUFFIX = ".delta"
90 DELTA_IMG = "DELTA_IMG"
95 VERBATIM_LIST = "Verbatim_List.txt"
100 SUPPORT_RENAME = "TRUE" #Use appropriate name
101 SUPPORT_CONTAINERS = "FALSE"
102 SUPPORT_DZIMAGE = "TRUE"
103 SUPPORT_VERBATIM = "TRUE"
108 logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
110 global GenerateDiffAttr
113 if len(sys.argv) < 5:
114 sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
115 UPDATE_TYPE = sys.argv[1]
116 PART_NAME = sys.argv[2] # lets make this also optional
118 BASE_OLD = sys.argv[3]
119 BASE_NEW = sys.argv[4]
120 OUT_DIR = sys.argv[5]
123 UPDATE_CFG_PATH = EMPTY
124 GenerateDiffAttr = "FALSE"
125 if UPDATE_TYPE == DELTA_FS:
126 #instead of arguments check it in outdirectory ?
127 if len(sys.argv) == 9:
128 ATTR_OLD = sys.argv[6]
129 ATTR_NEW = sys.argv[7]
130 UPDATE_CFG_PATH = '../'+sys.argv[8]
131 GenerateDiffAttr = "TRUE"
133 elif UPDATE_TYPE == DELTA_IMG or UPDATE_TYPE == FULL_IMG:
134 if len(sys.argv) == 7:
135 #Use path in better way
136 UPDATE_CFG_PATH = '../'+sys.argv[6]
139 global DIFFPATCH_UTIL
140 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
141 DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
142 DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
143 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
144 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
145 logging.info ('Diff Util Does NOT exist -- ABORT')
148 start = datetime.datetime.now().time()
149 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
150 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s] [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
152 ensure_dir_exists(OUT_DIR)
153 if GenerateDiffAttr == "TRUE":
154 if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
155 print >> sys.stderr, "Attributes missing -- ABORT"
159 # Should check if APT is supported on other linux flavours
161 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
162 logging.info ('Basic utils installed')
164 print >> sys.stderr, "Basic utils missing -- ABORT"
167 if UPDATE_TYPE == FULL_IMG:
168 SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
169 elif UPDATE_TYPE == DELTA_IMG:
170 SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
171 elif UPDATE_TYPE == DELTA_FS:
172 AttributeFile = ATTR_NEW
173 ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
174 Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
175 Old_files, Old_dirs = Get_Files(BASE_OLD)
176 New_files, New_dirs = Get_Files(BASE_NEW)
177 SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
179 if not UPDATE_CFG_PATH == EMPTY:
180 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
183 elif UPDATE_TYPE == EXTRA:
184 print('UPDATE_TYPE ---- EXTRA')
185 elif UPDATE_TYPE == PRE_UA:
186 print('UPDATE_TYPE ---- PRE_UA')
188 print('UPDATE_TYPE ---- UNKNOWN FORMAT')
190 if GenerateDiffAttr == "TRUE":
191 if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
194 end = datetime.datetime.now().time()
196 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
197 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
198 logging.info('Time start [%s] - Time end [%s]' % (start, end))
199 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
202 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
206 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
207 f = open(UPDATE_CFG_PATH, 'r')
208 lines = f.readlines()
210 f = open(UPDATE_CFG_PATH, 'w')
212 ConfigItems = line.split()
213 if ConfigItems[0] == DELTA_BIN:
214 DELTA = ConfigItems[1]
215 logging.info ('Updating %s config' % DELTA_BIN)
216 line = line.rstrip('\n')
218 line = line.replace(line, line+'\t'+str(Value)+'\n')
224 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
227 ZIMAGE_SCRIPT = COMMON_BIN_PATH+'./dzImagescript.sh'
228 ZIMAGE_OLD = BASE_OLD+'_unpacked'
229 ZIMAGE_NEW = BASE_NEW+'_unpacked'
230 DZIMAGE_HEADER = 'UnpackdzImage'
233 oldsize_d= os.path.getsize(BASE_OLD)
234 newsize_d= os.path.getsize(BASE_NEW)
235 SHA_BIN_DEST= hash_file(BASE_NEW)
236 SHA_BIN_BASE=hash_file(BASE_OLD)
238 #incase UPDATE CFG is empty
240 SS_UpdateSize(BASE_OLD, BASE_NEW)
241 #Should throw error if PART NAME NOT found??
242 if not UPDATE_CFG_PATH == EMPTY:
243 f = open(UPDATE_CFG_PATH, 'r')
244 lines = f.readlines()
246 f = open(UPDATE_CFG_PATH, 'w')
248 ConfigItems = line.split()
249 if ConfigItems[0] == DELTA_BIN:
250 logging.info ('Updating %s config' % DELTA_BIN)
251 DELTA = ConfigItems[1]
252 line = line.rstrip('\n')
253 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
259 #Any validation checks required?
260 if (DELTA_BIN == "zImage" or DELTA_BIN == "dzImage" or DELTA_BIN == "KERNEL" or DELTA_BIN == "BOOT" or DELTA_BIN == "boot" or DELTA_BIN == "recovery") and SUPPORT_DZIMAGE == "TRUE":
262 #Unpack Old and New Images for creating delta
263 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_OLD])
264 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_NEW])
267 Old_files, Old_dirs = Get_Files(ZIMAGE_OLD)
268 New_files, New_dirs = Get_Files(ZIMAGE_NEW)
270 patchLoc = '%s/%s_temp' % (OUT_DIR, DELTA_BIN)
271 ensure_dir_exists(patchLoc)
273 for elt in New_files:
275 src_file = ZIMAGE_OLD+'/'+elt
276 dst_file = ZIMAGE_NEW+'/'+elt
277 if not filecmp.cmp(src_file, dst_file):
278 patch = '%s/%s' % (patchLoc,elt)
279 DeltaFiles.append(patch)
280 subprocess.call([DIFF_UTIL,src_file,dst_file,patch])
281 logging.info('Make dz Image %s <--> %s ==> %s %s' % (src_file, dst_file , DELTA_BIN, patch))
283 #Append all delta files to make image.delta
285 #HEADER FORMAT MAGICNAME:FILECOUNT:[FILENAME:FILESIZE:][FILECONTENT/S]
286 HeaderStr = DZIMAGE_HEADER+DZIMAGE_SEP+'%d' % len(DeltaFiles)
287 HeaderStr = HeaderStr+DZIMAGE_SEP
289 with open(OUT_DIR+'/'+DELTA, 'w') as DeltaFile:
290 for fname in DeltaFiles:
291 DeltaSize = os.path.getsize(fname)
292 HeaderStr = HeaderStr+path_leaf(fname)+DZIMAGE_SEP+'%d' % DeltaSize
293 HeaderStr = HeaderStr+DZIMAGE_SEP
294 #Using 128 bytes as max Header.
295 logging.info('zImage Header - %s' % HeaderStr.ljust(128,'0'))
296 DeltaFile.write(HeaderStr.ljust(128,'0'))
297 for fname in DeltaFiles:
298 with open(fname) as infile:
299 DeltaFile.write(infile.read())
303 shutil.rmtree(patchLoc)
304 shutil.rmtree(ZIMAGE_OLD)
305 shutil.rmtree(ZIMAGE_NEW)
306 #Do we need to incorprate Max memory required for backup??
309 patchLoc = '%s/%s' % (OUT_DIR, DELTA)
310 logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
311 subprocess.call([DIFF_UTIL,BASE_OLD,BASE_NEW,patchLoc])
315 def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
316 logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
317 oldsize_d= os.path.getsize(BASE_OLD)
318 newsize_d= os.path.getsize(BASE_NEW)
319 SHA_BIN_DEST= hash_file(BASE_NEW)
320 SHA_BIN_BASE=hash_file(BASE_OLD)
321 #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
322 SS_UpdateSize(BASE_OLD, BASE_NEW)
324 if not UPDATE_CFG_PATH == EMPTY:
325 f = open(UPDATE_CFG_PATH, 'r')
326 lines = f.readlines()
328 f = open(UPDATE_CFG_PATH, 'w')
330 ConfigItems = line.split()
331 if ConfigItems[0] == DELTA_BIN:
332 logging.info ('Updating %s config' % DELTA_BIN)
333 DELTA = ConfigItems[1]
334 line = line.rstrip('\n')
335 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
341 def zipdir(path, zip):
342 for root, dirs, files in os.walk(path):
344 zip.write(os.path.join(root, file))
346 def ensure_dir_exists(path):
347 if not os.path.exists(path):
354 head, tail = ntpath.split(path) #This is for windows?? Recheck
358 head, tail = ntpath.split(path)
361 def difflines(list1, list2):
362 c = set(list1).union(set(list2))
363 d = set(list1).intersection(set(list2))
366 #Creating Diff between OLD and NEW attribute files v12
367 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
368 if GenerateDiffAttr == "FALSE":
370 with open(ATTR_OLD, 'r') as f_old:
371 lines1 = set(f_old.read().splitlines())
373 with open(ATTR_NEW, 'r') as f_new:
374 lines2 = set(f_new.read().splitlines())
376 lines = difflines(lines2, lines1)
377 with open(ATTR_FILE, 'w+') as file_out:
379 if line not in lines1:
380 logging.info('Diff_AttrFiles - %s' % line)
381 file_out.write(line+'\n')
389 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
390 #Full File Path should MATCH
391 if GenerateDiffAttr == "FALSE":
393 FilePath = '"/'+RequestedPath+'"'
394 #print ('FilePath - %s'% (FilePath))
395 with open(AttributeFile) as f:
398 if Type == SYMLINK_TYPE:
399 Sym_Attibutes.append(line)
401 File_Attibutes.append(line)
404 '''This function returns the SHA-1 hash of the file passed into it'''
405 def hash_file(filename):
410 # open file for reading in binary mode
411 with open(filename,'rb') as file:
412 # loop till the end of the file
415 # read only 1024 bytes at a time
416 chunk = file.read(1024*1024)
419 # return the hex representation of digest
422 def find_dupes_dir(BASE_OLD, BASE_NEW):
425 print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
426 logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
427 for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
428 #print('Scanning %s...' % rootbase)
429 for filename in fileListB:
430 path = os.path.join(rootbase, filename)
431 if os.path.islink(path):
434 file_hash = hash_file(path)
435 dups[file_hash] = path
437 for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
438 #print('Scanning %s...' % roottarget)
439 for filename in fileListT:
440 # Get the path to the file
441 path = os.path.join(roottarget, filename)
442 if os.path.islink(path):
445 file_hash = hash_file(path)
446 # Add or append the file path
447 if file_hash in dups:
448 BaseStr = dups.get(file_hash)
449 Baseloc = path.find('/')
450 TarLoc = BaseStr.find('/')
451 if not path[Baseloc:] == BaseStr[TarLoc:]:
452 logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
453 fdupes[path] = BaseStr
454 logging.info('Total Duplicate files %d' % (len(fdupes)))
458 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
461 print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
463 for filename in fileListB:
464 Src_File = BASE_OLD+'/'+filename
465 if os.path.islink(Src_File) or os.path.isdir(Src_File):
468 file_hash = hash_file(Src_File)
469 dups[file_hash] = Src_File
472 for filename in fileListT:
473 Dest_File = BASE_NEW+'/'+filename
474 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
477 file_hash = hash_file(Dest_File)
478 if file_hash in dups:
479 BaseStr = dups.get(file_hash)
480 Baseloc = BaseStr.find('/')
481 if not BaseStr[Baseloc:] == filename:
482 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
483 fdupes[BaseStr] = filename
485 logging.info('Total Duplicate files %d' % (len(fdupes)))
488 def SS_UpdateSize(src_file, dst_file):
491 oldsize_d= os.path.getsize(src_file)
492 newsize_d= os.path.getsize(dst_file)
493 if oldsize_d >= newsize_d:
503 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
504 print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
505 logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
507 # First let's fill up these categories
527 for elt in New_files:
528 if elt not in Old_files:
529 files_new.append(elt)
530 logging.info('New files %s' % elt)
532 # Generate Delete List
533 for elt in Old_files:
534 if elt not in New_files:
535 # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
536 files_removed.append(elt)
537 logging.info('Old files %s' % elt)
541 #print('List of Old Dirs %s' % elt)
542 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
543 if elt not in New_dirs:
544 Dir_removed.append(elt)
545 logging.info('Old Dirs %s' % elt+'/')
548 if elt not in Old_dirs:
549 Dir_Added.append(elt)
550 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
552 # What files have changed contents but not name/path?
553 for elt in New_files:
555 #Both are symbolic linkes and they differ
556 src_file = BASE_OLD+'/'+elt
557 dst_file = BASE_NEW+'/'+elt
558 #print('Files Changed - %s -%s' % (src_file,dst_file))
559 if os.path.islink(src_file) and os.path.islink(dst_file):
560 if not os.readlink(src_file) == os.readlink(dst_file):
561 files_changed.append(elt)
562 #print('%d Sym link files changed' % len(files_changed))
563 logging.info('Sym links Changed - %s' % elt)
565 files_unchanged.append(elt)
566 #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
567 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
568 if not filecmp.cmp(src_file, dst_file):
569 files_changed.append(elt)
570 #print('%d Normal files changed' % len(files_changed))
571 #print('Files Changed - %s' % elt)
573 files_unchanged.append(elt)
574 #File types differ between BASE and TARGET
576 logging.info('Files are of diff types but same names Src- %s Des- %s' % (src_file, dst_file))
577 #Both file types have changed and they differ
578 #Case 1: First Delete the OLD entry file type (Be it anything)
579 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
580 files_removed.append(elt)
581 files_new.append(elt)
584 # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
585 #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
587 print("Check for any verbatim under - %s" % VERBATIM_LIST)
588 if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
589 with open(VERBATIM_LIST, 'r') as F_News:
590 lines = set(F_News.read().splitlines())
592 if line in files_changed:
593 files_changed.remove(line)
594 files_removed.append(line)
595 if line in files_new:
596 files_new.remove(line)
598 #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
599 if SUPPORT_RENAME == "TRUE":
600 for elt in files_removed:
601 if os.path.isfile(BASE_OLD+'/'+elt):
602 FileName = path_leaf(elt)
603 entries = re.split('[0-9]' , FileName)
604 #Gives the STRING part of NAME. if name starts with version then later part wil b string
605 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
606 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
607 if len(entries[0]) > 0:
608 files_Del_List.update({entries[0]: elt})
610 for elt in files_new:
611 if os.path.isfile(BASE_NEW+'/'+elt):
612 FileName = path_leaf(elt)
613 entries = re.split('[0-9]' , FileName)
614 #print('Entires under NEWfiles list after split - %s %s - %s' % (FileName, entries[0], elt))
615 if len(entries[0]) > 0:
616 files_New_List.update({entries[0]: elt})
618 for key, value in files_Del_List.iteritems():
619 #print('Key value pair -%s -%s' % (key, value))
620 if key in files_New_List:
621 # this file is the same name in both!
622 src_file = BASE_OLD+'/'+value
623 dst_file = BASE_NEW+'/'+files_New_List[key]
624 olddirpath = path_head(files_New_List[key])
625 newdirpath = path_head(value)
626 if os.path.islink(src_file) or os.path.islink(dst_file):
627 logging.debug('Cannot diff as one of them is Symlink')
628 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
629 logging.debug('Cannot diff as one of them is dir')
631 #Pick the best diff of same type and diff names
632 files_renamed.append([files_New_List[key], value])
633 files_removed.remove(value)
634 files_new.remove(files_New_List[key])
638 Partition.txt contains Protocol for UPI
639 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
648 SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
649 Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
650 Partition_Doc_SymLinks = open(SymLinkDoc,'w')
652 print("writing diff'ed changed files...")
653 for elt in files_changed:
654 dst_file = BASE_NEW+'/'+elt
655 src_file = BASE_OLD+'/'+elt
656 #Both files are symbolic links and they differ
657 if os.path.islink(dst_file) and os.path.islink(src_file):
658 #Both are symlinks and they differ
659 logging.debug(' File Changed is Link %s ' % dst_file)
660 patch = os.readlink(dst_file)
661 Sym_Diff_Cnt = Sym_Diff_Cnt + 1
662 Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
663 Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
664 #Both are NORMAL files and they differ
665 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
666 #Both are files and they differ
667 Diff_Cnt = Diff_Cnt + 1
668 patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
669 patchLoc = '%s/%s' % (OUT_DIR, patchName)
670 logging.debug(' File Differ %s %s' % (src_file, dst_file))
671 SS_UpdateSize(src_file, dst_file)
674 ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
676 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
677 files_new.append(elt)
678 Diff_Cnt = Diff_Cnt - 1
680 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
682 Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
683 #Both differ but they are of diff types
685 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
686 files_removed.append(elt)
687 files_new.append(elt)
689 fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
690 for oldpath, newpath in fdupes.iteritems():
691 logging.info('Dupes %s -> %s' % (oldpath, newpath))
693 for elt in files_removed:
694 src_file = BASE_OLD+'/'+elt
695 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
696 if src_file in fdupes.keys():
697 dst_file = BASE_NEW+'/'+ fdupes[src_file]
698 logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
699 Move_Cnt = Move_Cnt + 1
700 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
701 files_removed.remove(elt)
702 files_new.remove(fdupes[src_file])
704 #Should be placed after removing duplicates, else they will be filtered here.
705 # loop shd b for all NEW files, rather than for all delete files (Current understanding)
706 # First Step: Sort & Filter out unwanted files
707 # Minimum condition used is,
708 # 1. File name should match 70%
709 # 2. Extensions should be same
710 # 3. File name length shd b greater than 3 char
711 # 4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
712 # 5. Should consider editdistance for RENAME LOGIC ==> TBD
714 Base_DelList = files_removed[:]
715 Base_NewList = files_new[:]
716 DelList = sorted(Base_DelList, key=path_leaf)
717 NewList = sorted(Base_NewList, key=path_leaf)
718 logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
722 #Remove unwanted items which we cant make diff with for rename logic
724 if os.path.islink(BASE_OLD+'/'+file):
726 elif os.path.isdir(BASE_OLD+'/'+file):
730 #logging.debug('Sorted del list - %s' % (file))
735 if os.path.islink(BASE_NEW+'/'+file):
737 elif os.path.isdir(BASE_NEW+'/'+file):
739 elif len(path_leaf(file)) <= 3:
740 logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
747 logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
749 for new_file in NewList:
751 DirPathNew = path_head(new_file)
752 FileNameNew = path_leaf(new_file)
754 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
755 New_fs = winning_patch_sz
758 for del_file in DelList:
759 FileNameOld = path_leaf(del_file)
760 if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
761 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
762 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
763 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
764 v1 = abs(New_fs-Del_fs)
765 v2 = (New_fs+Del_fs)/2
766 if( v2<=0 or ((v1/v2) * 100) > 30 ):
767 logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
769 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
771 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
772 if (DiffSize < 0.8 * winning_patch_sz):
773 winning_patch_sz = DiffSize
774 winning_file = del_file
775 elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
776 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
778 if len(winning_file) > 0:
779 logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
780 files_renamed.append([new_file, winning_file])
781 DelList.remove(winning_file)
782 files_removed.remove(winning_file)
783 files_new.remove(new_file)
785 #********************** Files should NOT be deleted for any such renames ***********************
787 if SUPPORT_RENAME == "TRUE":
788 for elt in files_renamed:
789 src_file = BASE_OLD+'/'+elt[1]
790 dst_file = BASE_NEW+'/'+elt[0]
791 Diff_Cnt = Diff_Cnt + 1
792 patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
793 #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
794 patchLoc = '%s/%s' % (OUT_DIR, patchName)
795 logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
796 # Should be careful of renaming files??
797 # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
798 # What if folder path has numerics??
800 if os.path.isdir(src_file) or os.path.isdir(dst_file):
801 #This case never occurs??
802 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
803 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
804 else: #Make sure these files are PROPER and they shd NOT be symlinks
805 if filecmp.cmp(src_file, dst_file):
806 Move_Cnt = Move_Cnt + 1
807 Diff_Cnt = Diff_Cnt - 1
808 Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
811 ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
813 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
814 files_new.append(elt)
815 Diff_Cnt = Diff_Cnt - 1
817 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
819 SS_UpdateSize(src_file, dst_file)
820 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
823 #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
824 #So that, the rename functionality will NOT create PATCH instead of verbatims.
826 if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
827 with open(VERBATIM_LIST, 'r') as F_News:
828 lines = set(F_News.read().splitlines())
830 if not line in files_new:
831 if os.path.exists(BASE_NEW+'/'+line):
832 files_new.append(line)
833 Verbatim_Cnt = Verbatim_Cnt+1
834 logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
838 for elt in files_removed:
839 #if files are part of patches after renaming, we shd remove them as part of removed.
840 src_file = BASE_OLD+'/'+elt
841 if os.path.islink(src_file):
842 Partition_Doc.write('DEL:SYM:%s\n' % (elt))
843 elif os.path.isdir(src_file):
844 #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
845 Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
847 Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
848 logging.debug(' File Deleted %s' % src_file)
849 Del_Cnt = Del_Cnt + 1
851 Dir_removed.sort(reverse=True)
852 for elt in Dir_removed:
853 #if Dir is empty, add it to the removed list.
854 src_file = BASE_OLD+'/'+elt
855 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
856 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
857 if os.path.isdir(src_file):
858 Partition_Doc.write('DEL:END:%s\n' % (elt))
859 Del_Cnt = Del_Cnt + 1
860 logging.debug(' Dir Deleted- %s' % src_file)
863 for elt in files_new:
864 dst_file = BASE_NEW+'/'+elt
865 newfiles_dest_path = 'system/'
866 ensure_dir_exists(newfiles_dest_path)
867 if os.path.islink(dst_file):
868 patch = os.readlink(dst_file)
869 logging.debug(' File New Links %s' % elt)
870 Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
871 #What if this is only a new sym link and folder already exists??? Should recheck
872 destpath = newfiles_dest_path + elt
873 if not os.path.exists(path_head(destpath)):
874 os.makedirs(path_head(destpath))
875 logging.info('New SymLink - Adding missing Dir')
876 #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
877 Sym_New_Cnt = Sym_New_Cnt + 1
878 elif os.path.isdir(dst_file): # We create just empty directory here
879 destpath = newfiles_dest_path + elt
880 if not os.path.exists(destpath):
881 os.makedirs(destpath)
882 logging.debug(' File New Dir %s' % destpath)
883 New_Cnt = New_Cnt + 1
885 New_Cnt = New_Cnt + 1
886 #newfiles_dest_path = OUT_DIR + '/system/'
887 destpath = newfiles_dest_path + elt
888 destdir = os.path.dirname(destpath)
889 logging.debug('New files - %s ==> %s' % (dst_file, destdir))
891 if not os.path.isdir(destdir):
895 logging.critical('Error in NEW files DIR entry -%s' % destdir)
899 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
900 shutil.copy2(dst_file, destpath)
901 logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
903 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
906 for elt in Dir_Added:
907 newfiles_dest_path = 'system/'
908 ensure_dir_exists(newfiles_dest_path)
909 destpath = newfiles_dest_path + elt
910 if not os.path.exists(destpath):
911 os.makedirs(destpath)
912 logging.debug(' DirList New Dir %s' % destpath)
913 New_Cnt = New_Cnt + 1
915 #Base directory should be system
916 print 'Compressing New files'
917 if (New_Cnt > 0 or Sym_New_Cnt > 0):
918 WorkingDir = os.getcwd()
919 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
920 logging.info('Curr Working Dir - %s' % os.getcwd())
921 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
922 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
923 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
924 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
926 shutil.rmtree(NEW_FILES_PATH)
927 # use 7z a system.7z ./*
929 #logging.info('%d Dir to be removed' % len(Dir_removed))
930 logging.info('%d files unchanged' % len(files_unchanged))
931 logging.info('%d files files_renamed' % len(files_renamed))
932 logging.info('%d files NEW' % len(files_new))
933 logging.info('%d File attr' % len(File_Attibutes))
934 logging.info('%d Sym attr' % len(Sym_Attibutes))
935 logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
936 print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
938 #There could be duplicates, TODO, can check before adding..
939 ATTR_FILE_D = open(ATTR_FILE,'a+')
940 for elt in File_Attibutes:
941 ATTR_FILE_D.write(elt)
942 for elt in Sym_Attibutes:
943 ATTR_FILE_D.write(elt)
947 Partition_Doc_SymLinks.close()
948 Partition_Read_SymLinks = open(SymLinkDoc,'r+')
949 Partition_Doc.write(Partition_Read_SymLinks.read())
950 Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
951 Partition_Doc_SymLinks.close()
952 Partition_Doc.close()
953 os.remove(SymLinkDoc)
955 if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
956 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
957 logging.info('No Delta Generated for %s' % PART_NAME)
958 shutil.rmtree(OUT_DIR)
961 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
963 #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
964 print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
965 shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
966 temp_apk = '../'+g_output_dir+'/'+b_apk
967 Patch = 'Patch_'+b_apk
968 ensure_dir_exists(Patch)
969 shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
971 #Size issue on Device side?? shd check this
972 subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
973 with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
974 lines = set(f_new.read().splitlines())
976 #print('Action ==> %s' % line)
977 #Action, Path, Patch = line.split('|')
978 Items = line.split('|')
981 ActualPath = a_folder+'/'+Path
982 PatchPath = Patch+'/'+Path
983 SrcPath = g_output_dir+'/'+path_leaf(Path)
984 #print('Action ==> %s Path ==> %s ' % (Action, Path))
986 patchName = g_output_dir+'/'+Items[2]
987 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
988 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
989 WorkingDir = os.getcwd()
990 os.chdir(WorkingDir+"/"+"temp_a")
991 subprocess.call(['cp', '--parents', Path, '../'+Patch])
994 WorkingDir = os.getcwd()
995 os.chdir(WorkingDir+"/"+"temp_a")
996 subprocess.call(['cp', '--parents', Path, '../'+Patch])
999 print('Apply_Container_Delta - Unknown Error')
1000 #print('Touch all files and set common attributes for DIFF generation')
1001 WorkingDir = os.getcwd()
1002 os.chdir(WorkingDir+"/"+Patch)
1004 CONTAINER_DATE = '200011111111.11'
1005 CONTAINER_MODE = '0755'
1006 subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
1007 subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
1008 subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
1010 print 'Update Intermediate Archive'
1011 #subprocess.call(['zip','-ryX', b_apk, '*'])
1012 subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
1013 os.chdir(WorkingDir)
1014 #print('Apply Path completed - Now create diff for this and place in patch folder')
1016 print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
1017 patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
1018 patchLoc = '%s/%s' % (g_output_dir, patchName)
1020 subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
1022 #Only on HOST... for testing
1023 if TEST_MODE == 'TRUE':
1024 UpgradedName = '%s_Upgraded' % (b_apk)
1025 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
1027 #This is file only with NEWS and empty diffs and same files.
1028 if TEST_MODE == 'FALSE':
1029 os.remove(g_output_dir+'/'+b_apk)
1030 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
1031 shutil.rmtree(Patch)
1033 def IsSymlink(info):
1034 return (info.external_attr >> 16) == 0120777
1036 def NewFiles(src, dest):
1038 subprocess.call(['cp','-rp', src,dest])
1040 #shutil.copytree(src, dest)
1041 #except OSError as e:
1042 # If the error was caused because the source wasn't a directory
1043 #if e.errno == errno.ENOTDIR:
1044 #shutil.copy2(src, dest)
1046 #print('Directory not copied. Error: %s' % e)
1048 def measure_two_filediffs(src, dst):
1049 patchLoc = 'temp.patch'
1050 subprocess.call([DIFF_UTIL,src,dst,patchLoc])
1051 result_size = os.path.getsize(patchLoc)
1055 def Get_Files(path):
1059 for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1060 for directory in directories:
1061 #DirName = os.path.join(root+'/',directory)
1062 DirName = os.path.join(root,directory)
1063 if os.path.islink(DirName):
1064 logging.debug('This is symlink pointing to dir -%s' % DirName)
1065 all_files.append(os.path.relpath(DirName, path))
1066 elif not os.listdir(DirName):
1067 #print('*****Empty Directory******* -%s', DirName)
1068 #This should NOT be appended ??? Empty dir shd b considered
1069 all_dirs.append(os.path.relpath(DirName, path))
1071 all_dirs.append(os.path.relpath(DirName, path))
1072 for filename in filenames:
1073 FileName = os.path.join(root,filename)
1074 all_files.append(os.path.relpath(FileName, path))
1078 return all_files, all_dirs
1081 USAGE_DOCSTRING = """
1082 Generate Delta using BASEOLD AND BASE NEW
1083 Attributes is optional
1084 Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1087 def Usage(docstring):
1088 print docstring.rstrip("\n")
1089 print COMMON_DOCSTRING
1093 if __name__ == '__main__':