mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49 def global_paths():
  50         global DIFF_UTIL
  51         global ZIPUTIL
  52         global NEW_FILES_PATH
  53         global NEW_FILES_FOLDER
  54         global NEW_FILES_ZIP_NAME
  55         global SYMLINK_TYPE
  56         global ATTR_DOC_EXT
  57         global SYMLINK_DOC_NAME
  58         global DIFF_PREFIX
  59         global DIFF_SUFFIX
  60         global SUPPORT_RENAME
  61         global NEW_PREFIX
  62         global DIFFPATCH_UTIL
  63         global SUPPORT_CONTAINERS
  64         global FULL_IMG
  65         global DELTA_IMG
  66         global DELTA_FS
  67         global EXTRA
  68         global COMMON_BIN_PATH
  69         global MEM_REQ
  70         global EMPTY
  71         global VERBATIM_LIST
  72         global MEM_FILE
  73
  74 COMMON_BIN_PATH = "../../common/bin/"
  75 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  76 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  77 #ZIPUTIL = "p7zip "
  78 ZIPUTIL = "7z -mf=off a system.7z "
  79 NEW_FILES_PATH = "system"
  80 NEW_FILES_FOLDER  = "system"
  81 NEW_FILES_ZIP_NAME = "system.7z"
  82 SYMLINK_TYPE = "SYM"
  83 ATTR_DOC_EXT = "_attr.txt"
  84 SYMLINK_DOC_NAME = "_sym.txt"
  85 PART_DOC_EXT = ".txt"
  86 DIFF_PREFIX = "diff"
  87 DIFF_SUFFIX = ".delta"
  88 NEW_PREFIX = 'new'
  89 FULL_IMG = "FULL_IMG"
  90 DELTA_IMG = "DELTA_IMG"
  91 DELTA_FS = "DELTA_FS"
  92 EXTRA = "EXTRA"
  93 PRE_UA = "PRE_UA"
  94 LOGFILE = "Delta.log"
  95 VERBATIM_LIST = "Verbatim_List.txt"
  96 EMPTY = ""
  97 MEM_REQ = 0
  98 MEM_FILE = "NULL"
  99
 100 SUPPORT_RENAME = "TRUE" #Use appropriate name
 101 SUPPORT_CONTAINERS = "FALSE"
 102 SUPPORT_VERBATIM = "TRUE"
 103
 104 TEST_MODE = "FALSE"
 105
 106 def main():
 107         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 108         global AttributeFile
 109         global GenerateDiffAttr
 110         try:
 111
 112                 if len(sys.argv) < 5:
 113                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 114                 UPDATE_TYPE = sys.argv[1]
 115                 PART_NAME = sys.argv[2]  # lets make this also optional
 116
 117                 BASE_OLD = sys.argv[3]
 118                 BASE_NEW = sys.argv[4]
 119                 OUT_DIR = sys.argv[5]
 120                 ATTR_OLD = EMPTY
 121                 ATTR_NEW = EMPTY
 122                 UPDATE_CFG_PATH = EMPTY
 123                 GenerateDiffAttr = "FALSE"
 124                 if UPDATE_TYPE == DELTA_FS:
 125                         #instead of arguments check it in outdirectory ?
 126                         if len(sys.argv) == 9:
 127                                 ATTR_OLD = sys.argv[6]
 128                                 ATTR_NEW = sys.argv[7]
 129                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 130                                 GenerateDiffAttr = "TRUE"
 131
 132                 elif UPDATE_TYPE == DELTA_IMG or UPDATE_TYPE == FULL_IMG:
 133                         if len(sys.argv) == 7:
 134                                 #Use path in better way
 135                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 136
 137                 global DIFF_UTIL
 138                 global DIFFPATCH_UTIL
 139                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 140                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 141                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 142                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 143                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 144                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 145                                 sys.exit(1)
 146
 147                 start = datetime.datetime.now().time()
 148                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 149                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 150
 151                 ensure_dir_exists(OUT_DIR)
 152                 if GenerateDiffAttr == "TRUE":
 153                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 154                                 print >> sys.stderr, "Attributes missing -- ABORT"
 155                                 sys.exit(1)
 156
 157
 158                 # Should check if APT is supported on other linux flavours
 159                 cache = apt.Cache()
 160                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 161                         logging.info ('Basic utils installed')
 162                 else:
 163                         print >> sys.stderr, "Basic utils missing -- ABORT"
 164                         sys.exit(1)
 165
 166                 if UPDATE_TYPE == FULL_IMG:
 167                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 168                 elif UPDATE_TYPE == DELTA_IMG:
 169                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 170                 elif UPDATE_TYPE == DELTA_FS:
 171                         AttributeFile = ATTR_NEW
 172                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 173                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 174                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 175                         New_files, New_dirs = Get_Files(BASE_NEW)
 176                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 177
 178                         if not UPDATE_CFG_PATH == EMPTY:
 179                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 180
 181
 182                 elif UPDATE_TYPE == EXTRA:
 183                         print('UPDATE_TYPE ---- EXTRA')
 184                 elif UPDATE_TYPE == PRE_UA:
 185                         print('UPDATE_TYPE ---- PRE_UA')
 186                 else:
 187                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 188
 189                 if GenerateDiffAttr == "TRUE":
 190                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 191                                 os.remove(ATTR_OLD)
 192                                 os.remove(ATTR_NEW)
 193                 end = datetime.datetime.now().time()
 194
 195                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 196                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 197                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 198                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 199
 200         except:
 201                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 202                 raise
 203
 204
 205 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 206         f = open(UPDATE_CFG_PATH, 'r')
 207         lines = f.readlines()
 208         f.close()
 209         f = open(UPDATE_CFG_PATH, 'w')
 210         for line in lines:
 211                 ConfigItems = line.split()
 212                 if ConfigItems[0] == DELTA_BIN:
 213                         DELTA = ConfigItems[1]
 214                         logging.info ('Updating %s config' % DELTA_BIN)
 215                         line = line.rstrip('\n')
 216                         Value = MEM_REQ
 217                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 218                         f.write(line)
 219                 else:
 220                         f.write(line)
 221         f.close()
 222
 223 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 224         #for sizes
 225
 226         oldsize_d= os.path.getsize(BASE_OLD)
 227         newsize_d= os.path.getsize(BASE_NEW)
 228         SHA_BIN_DEST= hash_file(BASE_NEW)
 229         SHA_BIN_BASE=hash_file(BASE_OLD)
 230
 231         #incase UPDATE CFG is empty
 232         DELTA = DELTA_BIN
 233         SS_UpdateSize(BASE_OLD, BASE_NEW)
 234         #Should throw error if PART NAME NOT found??
 235         if not UPDATE_CFG_PATH == EMPTY:
 236                 f = open(UPDATE_CFG_PATH, 'r')
 237                 lines = f.readlines()
 238                 f.close()
 239                 f = open(UPDATE_CFG_PATH, 'w')
 240                 for line in lines:
 241                         ConfigItems = line.split()
 242                         if ConfigItems[0] == DELTA_BIN:
 243                                 logging.info ('Updating %s config' % DELTA_BIN)
 244                                 DELTA = ConfigItems[1]
 245                                 line = line.rstrip('\n')
 246                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 247                                 f.write(line)
 248                         else:
 249                                 f.write(line)
 250                 f.close()
 251
 252         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 253         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 254         subprocess.call([DIFF_UTIL,BASE_OLD,BASE_NEW,patchLoc])
 255
 256
 257
 258 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 259         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 260         oldsize_d= os.path.getsize(BASE_OLD)
 261         newsize_d= os.path.getsize(BASE_NEW)
 262         SHA_BIN_DEST= hash_file(BASE_NEW)
 263         SHA_BIN_BASE=hash_file(BASE_OLD)
 264         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 265         SS_UpdateSize(BASE_OLD, BASE_NEW)
 266
 267         if not UPDATE_CFG_PATH == EMPTY:
 268                 f = open(UPDATE_CFG_PATH, 'r')
 269                 lines = f.readlines()
 270                 f.close()
 271                 f = open(UPDATE_CFG_PATH, 'w')
 272                 for line in lines:
 273                         ConfigItems = line.split()
 274                         if ConfigItems[0] == DELTA_BIN:
 275                                 logging.info ('Updating %s config' % DELTA_BIN)
 276                                 DELTA = ConfigItems[1]
 277                                 line = line.rstrip('\n')
 278                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 279                                 f.write(line)
 280                         else:
 281                                 f.write(line)
 282                 f.close()
 283
 284 def zipdir(path, zip):
 285     for root, dirs, files in os.walk(path):
 286         for file in files:
 287             zip.write(os.path.join(root, file))
 288
 289 def ensure_dir_exists(path):
 290         if not os.path.exists(path):
 291                 os.makedirs(path)
 292                 #shutil.rmtree(path)
 293         #os.makedirs(path)
 294
 295
 296 def path_leaf(path):
 297     head, tail = ntpath.split(path) #This is for windows?? Recheck
 298     return tail
 299
 300 def path_head(path):
 301     head, tail = ntpath.split(path)
 302     return head
 303
 304 def difflines(list1, list2):
 305     c = set(list1).union(set(list2))
 306     d = set(list1).intersection(set(list2))
 307     return list(c-d)
 308
 309 #Creating Diff between OLD and NEW attribute files v12
 310 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 311         if GenerateDiffAttr == "FALSE":
 312                 return
 313         with open(ATTR_OLD, 'r') as f_old:
 314                 lines1 = set(f_old.read().splitlines())
 315
 316         with open(ATTR_NEW, 'r') as f_new:
 317                 lines2 = set(f_new.read().splitlines())
 318
 319         lines = difflines(lines2, lines1)
 320         with open(ATTR_FILE, 'w+') as file_out:
 321                 for line in lines:
 322                         if line not in lines1:
 323                                 logging.info('Diff_AttrFiles - %s' % line)
 324                                 file_out.write(line+'\n')
 325
 326         f_new.close()
 327         f_old.close()
 328         file_out.close()
 329
 330
 331
 332 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 333         #Full File Path should MATCH
 334         if GenerateDiffAttr == "FALSE":
 335                 return
 336         FilePath = '"/'+RequestedPath+'"'
 337         #print ('FilePath - %s'% (FilePath))
 338         with open(AttributeFile) as f:
 339                 for line in f:
 340                         if FilePath in line:
 341                                 if Type == SYMLINK_TYPE:
 342                                         Sym_Attibutes.append(line)
 343                                 else:
 344                                         File_Attibutes.append(line)
 345
 346
 347 '''This function returns the SHA-1 hash of the file passed into it'''
 348 def hash_file(filename):
 349
 350    # make a hash object
 351    h = hashlib.sha1()
 352
 353    # open file for reading in binary mode
 354    with open(filename,'rb') as file:
 355        # loop till the end of the file
 356        chunk = 0
 357        while chunk != b'':
 358            # read only 1024 bytes at a time
 359            chunk = file.read(1024*1024)
 360            h.update(chunk)
 361
 362    # return the hex representation of digest
 363    return h.hexdigest()
 364
 365 def find_dupes_dir(BASE_OLD, BASE_NEW):
 366         dups = {}
 367         fdupes = {}
 368         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 369         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 370         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 371                 #print('Scanning %s...' % rootbase)
 372                 for filename in fileListB:
 373                         path = os.path.join(rootbase, filename)
 374                         if os.path.islink(path):
 375                                 continue
 376                         # Calculate hash
 377                         file_hash = hash_file(path)
 378                         dups[file_hash] = path
 379
 380         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 381                 #print('Scanning %s...' % roottarget)
 382                 for filename in fileListT:
 383                         # Get the path to the file
 384                         path = os.path.join(roottarget, filename)
 385                         if os.path.islink(path):
 386                                 continue
 387                         # Calculate hash
 388                         file_hash = hash_file(path)
 389                         # Add or append the file path
 390                         if file_hash in dups:
 391                                 BaseStr = dups.get(file_hash)
 392                                 Baseloc = path.find('/')
 393                                 TarLoc = BaseStr.find('/')
 394                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 395                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 396                                         fdupes[path] = BaseStr
 397         logging.info('Total Duplicate files %d' % (len(fdupes)))
 398         return fdupes
 399
 400
 401 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 402         dups = {}
 403         fdupes = {}
 404         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 405
 406         for filename in fileListB:
 407                 Src_File = BASE_OLD+'/'+filename
 408                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 409                         continue
 410                 # Calculate hash
 411                 file_hash = hash_file(Src_File)
 412                 dups[file_hash] = Src_File
 413
 414
 415         for filename in fileListT:
 416                 Dest_File = BASE_NEW+'/'+filename
 417                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 418                         continue
 419                 # Calculate hash
 420                 file_hash = hash_file(Dest_File)
 421                 if file_hash in dups:
 422                         BaseStr = dups.get(file_hash)
 423                         Baseloc = BaseStr.find('/')
 424                         if not BaseStr[Baseloc:] == filename:
 425                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 426                                 fdupes[BaseStr] = filename
 427
 428         logging.info('Total Duplicate files %d' % (len(fdupes)))
 429         return fdupes
 430
 431 def SS_UpdateSize(src_file, dst_file):
 432         global MEM_REQ
 433         global MEM_FILE
 434         oldsize_d= os.path.getsize(src_file)
 435         newsize_d= os.path.getsize(dst_file)
 436         if oldsize_d >= newsize_d:
 437                 Max = newsize_d
 438         else:
 439                 Max = oldsize_d
 440         if MEM_REQ < Max:
 441                 MEM_REQ = Max
 442                 MEM_FILE = dst_file
 443
 444
 445
 446 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 447         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 448         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 449
 450         # First let's fill up these categories
 451         files_new = []
 452         files_removed = []
 453         Dir_removed = []
 454         Dir_Added = []
 455         files_changed = []
 456         files_unchanged = []
 457         files_renamed = []
 458         File_Attibutes = []
 459         Sym_Attibutes = []
 460
 461         files_Del_List = {}
 462         files_New_List = {}
 463         MyDict_Patches = {}
 464
 465
 466
 467         PWD = os.getcwd()
 468
 469         # Generate NEW List
 470         for elt in New_files:
 471                 if elt not in Old_files:
 472                         files_new.append(elt)
 473                         logging.info('New files %s' % elt)
 474
 475         # Generate Delete List
 476         for elt in Old_files:
 477                 if elt not in New_files:
 478                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 479                         files_removed.append(elt)
 480                         logging.info('Old files %s' % elt)
 481
 482
 483         for elt in Old_dirs:
 484                 #print('List of Old Dirs %s' % elt)
 485                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 486                 if elt not in New_dirs:
 487                         Dir_removed.append(elt)
 488                         logging.info('Old Dirs %s' % elt+'/')
 489
 490         for elt in New_dirs:
 491                 if elt not in Old_dirs:
 492                         Dir_Added.append(elt)
 493                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 494
 495         # What files have changed contents but not name/path?
 496         for elt in New_files:
 497                 if elt in Old_files:
 498                         #Both are symbolic linkes and they differ
 499                         src_file = BASE_OLD+'/'+elt
 500                         dst_file = BASE_NEW+'/'+elt
 501                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 502                         if os.path.islink(src_file) and os.path.islink(dst_file):
 503                                 if not os.readlink(src_file) == os.readlink(dst_file):
 504                                         files_changed.append(elt)
 505                                         #print('%d Sym link files changed' % len(files_changed))
 506                                         logging.info('Sym links Changed - %s' % elt)
 507                                 else:
 508                                         files_unchanged.append(elt)
 509                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 510                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 511                                 if not filecmp.cmp(src_file, dst_file):
 512                                         files_changed.append(elt)
 513                                         #print('%d Normal files changed' % len(files_changed))
 514                                         #print('Files Changed - %s' % elt)
 515                                 else:
 516                                         files_unchanged.append(elt)
 517                         #File types differ between BASE and TARGET
 518                         else:
 519                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 520                                 #Both file types have changed and they differ
 521                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 522                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 523                                 files_removed.append(elt)
 524                                 files_new.append(elt)
 525
 526
 527         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 528         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 529
 530         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 531         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 532                 with open(VERBATIM_LIST, 'r') as F_News:
 533                         lines = set(F_News.read().splitlines())
 534                 for line in lines:
 535                         if line in files_changed:
 536                                 files_changed.remove(line)
 537                                 files_removed.append(line)
 538                         if line in files_new:
 539                                 files_new.remove(line)
 540
 541         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 542         if SUPPORT_RENAME == "TRUE":
 543                 for elt in files_removed:
 544                         if os.path.isfile(BASE_OLD+'/'+elt):
 545                                 FileName = path_leaf(elt)
 546                                 entries = re.split('[0-9]' , FileName)
 547                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 548                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 549                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 550                                 if len(entries[0]) > 0:
 551                                         files_Del_List.update({entries[0]: elt})
 552
 553                 for elt in files_new:
 554                         if os.path.isfile(BASE_NEW+'/'+elt):
 555                                 FileName = path_leaf(elt)
 556                                 entries = re.split('[0-9]' , FileName)
 557                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 558                                 if len(entries[0]) > 0:
 559                                         files_New_List.update({entries[0]: elt})
 560
 561                 for key, value in files_Del_List.iteritems():
 562                         #print('Key value pair -%s -%s' % (key, value))
 563                         if key in files_New_List:
 564                                 # this file is the same name in both!
 565                                 src_file = BASE_OLD+'/'+value
 566                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 567                                 olddirpath = path_head(files_New_List[key])
 568                                 newdirpath = path_head(value)
 569                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 570                                         logging.debug('Cannot diff as one of them is Symlink')
 571                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 572                                         logging.debug('Cannot diff as one of them is dir')
 573                                 else:
 574                                         #Pick the best diff of same type and diff names
 575                                         files_renamed.append([files_New_List[key], value])
 576                                         files_removed.remove(value)
 577                                         files_new.remove(files_New_List[key])
 578
 579         '''
 580         Patch Section
 581                 Partition.txt contains Protocol for UPI
 582                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 583         '''
 584         Sym_Diff_Cnt = 0
 585         Sym_New_Cnt = 0;
 586         Del_Cnt = 0
 587         New_Cnt = 0
 588         Diff_Cnt = 0
 589         Move_Cnt = 0
 590         Verbatim_Cnt = 0
 591         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 592         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 593         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 594
 595         print("writing diff'ed changed files...")
 596         for elt in files_changed:
 597                 dst_file = BASE_NEW+'/'+elt
 598                 src_file = BASE_OLD+'/'+elt
 599                 #Both files are symbolic links and they differ
 600                 if os.path.islink(dst_file) and os.path.islink(src_file):
 601                 #Both are symlinks and they differ
 602                         logging.debug(' File Changed is Link %s ' % dst_file)
 603                         patch = os.readlink(dst_file)
 604                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 605                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 606                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 607                 #Both are NORMAL files and they differ
 608                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 609                         #Both are files and they differ
 610                         Diff_Cnt = Diff_Cnt + 1
 611                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 612                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 613                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 614                         SS_UpdateSize(src_file, dst_file)
 615
 616                         FORMAT = "REG"
 617                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 618                         if ret is not 0:
 619                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 620                                 files_new.append(elt)
 621                                 Diff_Cnt = Diff_Cnt - 1
 622                         else:
 623                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 624
 625                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 626                 #Both differ but they are of diff types
 627                 else:
 628                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 629                         files_removed.append(elt)
 630                         files_new.append(elt)
 631
 632         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 633         for oldpath, newpath in fdupes.iteritems():
 634                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 635
 636         for elt in files_removed:
 637                 src_file = BASE_OLD+'/'+elt
 638                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 639                 if src_file in fdupes.keys():
 640                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 641                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 642                         Move_Cnt = Move_Cnt + 1
 643                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 644                         files_removed.remove(elt)
 645                         files_new.remove(fdupes[src_file])
 646
 647         #Should be placed after removing duplicates, else they will be filtered here.
 648         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 649         #       First Step: Sort & Filter out unwanted files
 650         # Minimum condition used is,
 651         #       1. File name should match 70%
 652         #       2. Extensions should be same
 653         #       3. File name length shd b greater than 3 char
 654         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 655         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 656
 657         Base_DelList = files_removed[:]
 658         Base_NewList = files_new[:]
 659         DelList = sorted(Base_DelList, key=path_leaf)
 660         NewList = sorted(Base_NewList, key=path_leaf)
 661         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 662
 663         Filter1 = []
 664         Filter2 = []
 665         #Remove unwanted items which we cant make diff with for rename logic
 666         for file in DelList:
 667                 if os.path.islink(BASE_OLD+'/'+file):
 668                         continue
 669                 elif os.path.isdir(BASE_OLD+'/'+file):
 670                         continue
 671                 else:
 672                         Filter1.append(file)
 673                         #logging.debug('Sorted del list - %s' % (file))
 674
 675         DelList = Filter1
 676
 677         for file in NewList:
 678                 if os.path.islink(BASE_NEW+'/'+file):
 679                         continue
 680                 elif os.path.isdir(BASE_NEW+'/'+file):
 681                         continue
 682                 elif len(path_leaf(file)) <= 3:
 683                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 684                         continue
 685                 else:
 686                         Filter2.append(file)
 687
 688         NewList = Filter2
 689
 690         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 691
 692         for new_file in NewList:
 693                 R_Flag = 'FALSE';
 694                 DirPathNew = path_head(new_file)
 695                 FileNameNew = path_leaf(new_file)
 696                 DiffSize = 0
 697                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 698                 New_fs = winning_patch_sz
 699                 winning_file = ''
 700
 701                 for del_file in DelList:
 702                         FileNameOld = path_leaf(del_file)
 703                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 704                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 705                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 706                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 707                                 v1 = abs(New_fs-Del_fs)
 708                                 v2 = (New_fs+Del_fs)/2
 709                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 710                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 711                                         continue
 712                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 713                                 R_Flag = 'TRUE';
 714                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 715                                 if (DiffSize < 0.8 * winning_patch_sz):
 716                                         winning_patch_sz = DiffSize
 717                                         winning_file = del_file
 718                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 719                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 720                                 break;
 721                 if len(winning_file) > 0:
 722                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 723                         files_renamed.append([new_file, winning_file])
 724                         DelList.remove(winning_file)
 725                         files_removed.remove(winning_file)
 726                         files_new.remove(new_file)
 727
 728         #********************** Files should NOT be deleted for any such renames ***********************
 729
 730         if SUPPORT_RENAME == "TRUE":
 731                 for elt in files_renamed:
 732                         src_file = BASE_OLD+'/'+elt[1]
 733                         dst_file = BASE_NEW+'/'+elt[0]
 734                         Diff_Cnt = Diff_Cnt + 1
 735                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 736                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 737                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 738                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 739                         # Should be careful of renaming files??
 740                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 741                         # What if folder path has numerics??
 742
 743                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 744                                 #This case never occurs??
 745                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 746                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 747                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 748                                 if filecmp.cmp(src_file, dst_file):
 749                                         Move_Cnt = Move_Cnt + 1
 750                                         Diff_Cnt = Diff_Cnt - 1
 751                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 752                                 else:
 753                                         FORMAT = "REG"
 754                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 755                                         if ret is not 0:
 756                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 757                                                 files_new.append(elt)
 758                                                 Diff_Cnt = Diff_Cnt - 1
 759                                         else:
 760                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 761
 762                                 SS_UpdateSize(src_file, dst_file)
 763                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 764
 765
 766         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 767         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 768
 769         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 770                 with open(VERBATIM_LIST, 'r') as F_News:
 771                         lines = set(F_News.read().splitlines())
 772                 for line in lines:
 773                         if not line in files_new:
 774                                 if os.path.exists(BASE_NEW+'/'+line):
 775                                         files_new.append(line)
 776                                         Verbatim_Cnt = Verbatim_Cnt+1
 777                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 778
 779
 780
 781         for elt in files_removed:
 782                 #if files are part of patches after renaming, we shd remove them as part of removed.
 783                 src_file = BASE_OLD+'/'+elt
 784                 if os.path.islink(src_file):
 785                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 786                 elif os.path.isdir(src_file):
 787                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 788                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 789                 else:
 790                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 791                 logging.debug(' File Deleted %s' % src_file)
 792                 Del_Cnt = Del_Cnt + 1
 793
 794         Dir_removed.sort(reverse=True)
 795         for elt in Dir_removed:
 796                 #if Dir is empty, add it to the removed list.
 797                 src_file = BASE_OLD+'/'+elt
 798                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 799                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 800                 if os.path.isdir(src_file):
 801                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 802                         Del_Cnt = Del_Cnt + 1
 803                         logging.debug(' Dir Deleted- %s' % src_file)
 804
 805
 806         for elt in files_new:
 807                 dst_file = BASE_NEW+'/'+elt
 808                 newfiles_dest_path = 'system/'
 809                 ensure_dir_exists(newfiles_dest_path)
 810                 if os.path.islink(dst_file):
 811                         patch = os.readlink(dst_file)
 812                         logging.debug(' File New Links %s' % elt)
 813                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 814                         #What if this is only a new sym link and folder already exists??? Should recheck
 815                         destpath = newfiles_dest_path + elt
 816                         if not os.path.exists(path_head(destpath)):
 817                                 os.makedirs(path_head(destpath))
 818                                 logging.info('New SymLink - Adding missing Dir')
 819                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 820                         Sym_New_Cnt = Sym_New_Cnt + 1
 821                 elif os.path.isdir(dst_file): # We create just empty directory here
 822                         destpath = newfiles_dest_path + elt
 823                         if not os.path.exists(destpath):
 824                                 os.makedirs(destpath)
 825                                 logging.debug(' File New Dir %s' % destpath)
 826                                 New_Cnt = New_Cnt + 1
 827                 else:
 828                         New_Cnt = New_Cnt + 1
 829                         #newfiles_dest_path = OUT_DIR + '/system/'
 830                         destpath = newfiles_dest_path + elt
 831                         destdir = os.path.dirname(destpath)
 832                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 833
 834                         if not os.path.isdir(destdir):
 835                                 try:
 836                                         os.makedirs(destdir)
 837                                 except:
 838                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 839                                         raise
 840
 841                         try:
 842                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 843                                         shutil.copy2(dst_file, destpath)
 844                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 845                         except:
 846                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 847                                 raise
 848
 849         for elt in Dir_Added:
 850                 newfiles_dest_path = 'system/'
 851                 ensure_dir_exists(newfiles_dest_path)
 852                 destpath = newfiles_dest_path + elt
 853                 if not os.path.exists(destpath):
 854                         os.makedirs(destpath)
 855                         logging.debug(' DirList New Dir %s' % destpath)
 856                         New_Cnt = New_Cnt + 1
 857
 858         #Base directory should be system
 859         print 'Compressing New files'
 860         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 861                 WorkingDir = os.getcwd()
 862                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 863                 logging.info('Curr Working Dir - %s' % os.getcwd())
 864                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 865                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 866                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 867                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 868                 os.chdir(WorkingDir)
 869                 shutil.rmtree(NEW_FILES_PATH)
 870                 # use 7z a system.7z ./*
 871
 872         #logging.info('%d Dir to be removed' % len(Dir_removed))
 873         logging.info('%d files unchanged' % len(files_unchanged))
 874         logging.info('%d files files_renamed' % len(files_renamed))
 875         logging.info('%d files NEW' % len(files_new))
 876         logging.info('%d File attr' % len(File_Attibutes))
 877         logging.info('%d Sym attr' % len(Sym_Attibutes))
 878         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 879         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 880
 881         #There could be duplicates, TODO, can check before adding..
 882         ATTR_FILE_D = open(ATTR_FILE,'a+')
 883         for elt in File_Attibutes:
 884                 ATTR_FILE_D.write(elt)
 885         for elt in Sym_Attibutes:
 886                 ATTR_FILE_D.write(elt)
 887
 888         ATTR_FILE_D.close()
 889
 890         Partition_Doc_SymLinks.close()
 891         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 892         Partition_Doc.write(Partition_Read_SymLinks.read())
 893         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 894         Partition_Doc_SymLinks.close()
 895         Partition_Doc.close()
 896         os.remove(SymLinkDoc)
 897
 898         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 899                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 900                 logging.info('No Delta Generated for %s' % PART_NAME)
 901                 shutil.rmtree(OUT_DIR)
 902
 903
 904 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 905
 906         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 907         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 908         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 909         temp_apk = '../'+g_output_dir+'/'+b_apk
 910         Patch = 'Patch_'+b_apk
 911         ensure_dir_exists(Patch)
 912         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 913
 914         #Size issue on Device side?? shd check this
 915         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 916         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 917                 lines = set(f_new.read().splitlines())
 918                 for line in lines:
 919                         #print('Action ==> %s' % line)
 920                         #Action, Path, Patch = line.split('|')
 921                         Items = line.split('|')
 922                         Action = Items[0]
 923                         Path = Items[1]
 924                         ActualPath = a_folder+'/'+Path
 925                         PatchPath = Patch+'/'+Path
 926                         SrcPath = g_output_dir+'/'+path_leaf(Path)
 927                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 928                         if line[0] == 'c':
 929                                 patchName = g_output_dir+'/'+Items[2]
 930                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 931                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
 932                                 WorkingDir = os.getcwd()
 933                                 os.chdir(WorkingDir+"/"+"temp_a")
 934                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 935                                 os.chdir(WorkingDir)
 936                         elif line[0] == 's':
 937                                 WorkingDir = os.getcwd()
 938                                 os.chdir(WorkingDir+"/"+"temp_a")
 939                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 940                                 os.chdir(WorkingDir)
 941                         else:
 942                                 print('Apply_Container_Delta - Unknown Error')
 943         #print('Touch all files and set common attributes for DIFF generation')
 944         WorkingDir = os.getcwd()
 945         os.chdir(WorkingDir+"/"+Patch)
 946
 947         CONTAINER_DATE = '200011111111.11'
 948         CONTAINER_MODE = '0755'
 949         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
 950         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
 951         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
 952
 953         print 'Update Intermediate Archive'
 954         #subprocess.call(['zip','-ryX', b_apk, '*'])
 955         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
 956         os.chdir(WorkingDir)
 957         #print('Apply Path completed - Now create diff for this and place in patch folder')
 958         #print os.getcwd()
 959         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
 960         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
 961         patchLoc = '%s/%s' % (g_output_dir, patchName)
 962
 963         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
 964
 965         #Only on HOST... for testing
 966         if TEST_MODE == 'TRUE':
 967                 UpgradedName = '%s_Upgraded' % (b_apk)
 968                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
 969
 970         #This is file only with NEWS and empty diffs and same files.
 971         if TEST_MODE == 'FALSE':
 972                 os.remove(g_output_dir+'/'+b_apk)
 973                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
 974                 shutil.rmtree(Patch)
 975
 976 def IsSymlink(info):
 977   return (info.external_attr >> 16) == 0120777
 978
 979 def NewFiles(src, dest):
 980         print src,dest
 981         subprocess.call(['cp','-rp', src,dest])
 982     #try:
 983                 #shutil.copytree(src, dest)
 984     #except OSError as e:
 985         # If the error was caused because the source wasn't a directory
 986         #if e.errno == errno.ENOTDIR:
 987             #shutil.copy2(src, dest)
 988         #else:
 989             #print('Directory not copied. Error: %s' % e)
 990
 991 def measure_two_filediffs(src, dst):
 992         patchLoc = 'temp.patch'
 993         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
 994         result_size = os.path.getsize(patchLoc)
 995         os.remove(patchLoc)
 996         return result_size
 997
 998 def Get_Files(path):
 999         all_files = []
1000         all_dirs = []
1001
1002         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1003                 for directory in directories:
1004                         #DirName = os.path.join(root+'/',directory)
1005                         DirName = os.path.join(root,directory)
1006                         if os.path.islink(DirName):
1007                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1008                                 all_files.append(os.path.relpath(DirName, path))
1009                         elif not os.listdir(DirName):
1010                                 #print('*****Empty Directory******* -%s', DirName)
1011                                 #This should NOT be appended ??? Empty dir shd b considered
1012                                 all_dirs.append(os.path.relpath(DirName, path))
1013                         else:
1014                                 all_dirs.append(os.path.relpath(DirName, path))
1015                 for filename in filenames:
1016                         FileName = os.path.join(root,filename)
1017                         all_files.append(os.path.relpath(FileName, path))
1018
1019         all_files.sort()
1020         all_dirs.sort()
1021         return all_files, all_dirs
1022
1023
1024 USAGE_DOCSTRING = """
1025       Generate Delta using BASEOLD AND BASE NEW
1026           Attributes is optional
1027           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1028 """
1029
1030 def Usage(docstring):
1031   print docstring.rstrip("\n")
1032   print COMMON_DOCSTRING
1033
1034
1035
1036 if __name__ == '__main__':
1037         main()
1038