mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49 def global_paths():
  50         global DIFF_UTIL
  51         global ZIPUTIL
  52         global NEW_FILES_PATH
  53         global NEW_FILES_ZIP_NAME
  54         global SYMLINK_TYPE
  55         global ATTR_DOC_EXT
  56         global SYMLINK_DOC_NAME
  57         global DIFF_PREFIX
  58         global DIFF_SUFFIX
  59         global SUPPORT_RENAME
  60         global NEW_PREFIX
  61         global DIFFPATCH_UTIL
  62         global SUPPORT_CONTAINERS
  63         global FULL_IMG
  64         global DELTA_IMG
  65         global DELTA_FS
  66         global EXTRA
  67         global COMMON_BIN_PATH
  68         global MEM_REQ
  69         global EMPTY
  70         global VERBATIM_LIST
  71         global MEM_FILE
  72
  73 COMMON_BIN_PATH = "../../common/bin/"
  74 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  75 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  76 #ZIPUTIL = "p7zip "
  77 ZIPUTIL = "7z -mf=off a system.7z "
  78 NEW_FILES_PATH = "run/upgrade-sysroot"
  79 NEW_FILES_ZIP_NAME = "system.7z"
  80 SYMLINK_TYPE = "SYM"
  81 ATTR_DOC_EXT = "_attr.txt"
  82 SYMLINK_DOC_NAME = "_sym.txt"
  83 PART_DOC_EXT = ".txt"
  84 DIFF_PREFIX = "diff"
  85 DIFF_SUFFIX = ".delta"
  86 NEW_PREFIX = 'new'
  87 FULL_IMG = "FULL_IMG"
  88 DELTA_IMG = "DELTA_IMG"
  89 DELTA_FS = "DELTA_FS"
  90 EXTRA = "EXTRA"
  91 PRE_UA = "PRE_UA"
  92 LOGFILE = "Delta.log"
  93 VERBATIM_LIST = "Verbatim_List.txt"
  94 EMPTY = ""
  95 MEM_REQ = 0
  96 MEM_FILE = "NULL"
  97
  98 SUPPORT_RENAME = "TRUE" #Use appropriate name
  99 SUPPORT_CONTAINERS = "FALSE"
 100 SUPPORT_VERBATIM = "TRUE"
 101
 102 TEST_MODE = "FALSE"
 103
 104 def main():
 105         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 106         global AttributeFile
 107         global GenerateDiffAttr
 108         try:
 109
 110                 if len(sys.argv) < 5:
 111                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 112                 UPDATE_TYPE = sys.argv[1]
 113                 PART_NAME = sys.argv[2]  # lets make this also optional
 114
 115                 BASE_OLD = sys.argv[3]
 116                 BASE_NEW = sys.argv[4]
 117                 OUT_DIR = sys.argv[5]
 118                 ATTR_OLD = EMPTY
 119                 ATTR_NEW = EMPTY
 120                 UPDATE_CFG_PATH = EMPTY
 121                 GenerateDiffAttr = "FALSE"
 122                 if UPDATE_TYPE == DELTA_FS:
 123                         #instead of arguments check it in outdirectory ?
 124                         if len(sys.argv) == 9:
 125                                 ATTR_OLD = sys.argv[6]
 126                                 ATTR_NEW = sys.argv[7]
 127                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 128                                 GenerateDiffAttr = "TRUE"
 129
 130                 elif UPDATE_TYPE == DELTA_IMG or UPDATE_TYPE == FULL_IMG:
 131                         if len(sys.argv) == 7:
 132                                 #Use path in better way
 133                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 134
 135                 global DIFF_UTIL
 136                 global DIFFPATCH_UTIL
 137                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 138                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 139                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 140                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 141                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 142                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 143                                 sys.exit(1)
 144
 145                 start = datetime.datetime.now().time()
 146                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 147                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 148
 149                 ensure_dir_exists(OUT_DIR)
 150                 if GenerateDiffAttr == "TRUE":
 151                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 152                                 print >> sys.stderr, "Attributes missing -- ABORT"
 153                                 sys.exit(1)
 154
 155
 156                 # Should check if APT is supported on other linux flavours
 157                 cache = apt.Cache()
 158                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 159                         logging.info ('Basic utils installed')
 160                 else:
 161                         print >> sys.stderr, "Basic utils missing -- ABORT"
 162                         sys.exit(1)
 163
 164                 if UPDATE_TYPE == FULL_IMG:
 165                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 166                 elif UPDATE_TYPE == DELTA_IMG:
 167                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 168                 elif UPDATE_TYPE == DELTA_FS:
 169                         AttributeFile = ATTR_NEW
 170                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 171                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 172                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 173                         New_files, New_dirs = Get_Files(BASE_NEW)
 174                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 175
 176                         if not UPDATE_CFG_PATH == EMPTY:
 177                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 178
 179
 180                 elif UPDATE_TYPE == EXTRA:
 181                         print('UPDATE_TYPE ---- EXTRA')
 182                 elif UPDATE_TYPE == PRE_UA:
 183                         print('UPDATE_TYPE ---- PRE_UA')
 184                 else:
 185                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 186
 187                 if GenerateDiffAttr == "TRUE":
 188                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 189                                 os.remove(ATTR_OLD)
 190                                 os.remove(ATTR_NEW)
 191                 end = datetime.datetime.now().time()
 192
 193                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 194                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 195                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 196                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 197
 198         except:
 199                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 200                 raise
 201
 202
 203 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 204         f = open(UPDATE_CFG_PATH, 'r')
 205         lines = f.readlines()
 206         f.close()
 207         f = open(UPDATE_CFG_PATH, 'w')
 208         for line in lines:
 209                 ConfigItems = line.split()
 210                 if ConfigItems[0] == DELTA_BIN:
 211                         DELTA = ConfigItems[1]
 212                         logging.info ('Updating %s config' % DELTA_BIN)
 213                         line = line.rstrip('\n')
 214                         Value = MEM_REQ
 215                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 216                         f.write(line)
 217                 else:
 218                         f.write(line)
 219         f.close()
 220
 221 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 222         #for sizes
 223
 224         oldsize_d= os.path.getsize(BASE_OLD)
 225         newsize_d= os.path.getsize(BASE_NEW)
 226         SHA_BIN_DEST= hash_file(BASE_NEW)
 227         SHA_BIN_BASE=hash_file(BASE_OLD)
 228
 229         #incase UPDATE CFG is empty
 230         DELTA = DELTA_BIN
 231         SS_UpdateSize(BASE_OLD, BASE_NEW)
 232         #Should throw error if PART NAME NOT found??
 233         if not UPDATE_CFG_PATH == EMPTY:
 234                 f = open(UPDATE_CFG_PATH, 'r')
 235                 lines = f.readlines()
 236                 f.close()
 237                 f = open(UPDATE_CFG_PATH, 'w')
 238                 for line in lines:
 239                         ConfigItems = line.split()
 240                         if ConfigItems[0] == DELTA_BIN:
 241                                 logging.info ('Updating %s config' % DELTA_BIN)
 242                                 DELTA = ConfigItems[1]
 243                                 line = line.rstrip('\n')
 244                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 245                                 f.write(line)
 246                         else:
 247                                 f.write(line)
 248                 f.close()
 249
 250         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 251         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 252         subprocess.call([DIFF_UTIL,BASE_OLD,BASE_NEW,patchLoc])
 253
 254
 255
 256 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 257         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 258         oldsize_d= os.path.getsize(BASE_OLD)
 259         newsize_d= os.path.getsize(BASE_NEW)
 260         SHA_BIN_DEST= hash_file(BASE_NEW)
 261         SHA_BIN_BASE=hash_file(BASE_OLD)
 262         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 263         SS_UpdateSize(BASE_OLD, BASE_NEW)
 264
 265         if not UPDATE_CFG_PATH == EMPTY:
 266                 f = open(UPDATE_CFG_PATH, 'r')
 267                 lines = f.readlines()
 268                 f.close()
 269                 f = open(UPDATE_CFG_PATH, 'w')
 270                 for line in lines:
 271                         ConfigItems = line.split()
 272                         if ConfigItems[0] == DELTA_BIN:
 273                                 logging.info ('Updating %s config' % DELTA_BIN)
 274                                 DELTA = ConfigItems[1]
 275                                 line = line.rstrip('\n')
 276                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 277                                 f.write(line)
 278                         else:
 279                                 f.write(line)
 280                 f.close()
 281
 282 def zipdir(path, zip):
 283     for root, dirs, files in os.walk(path):
 284         for file in files:
 285             zip.write(os.path.join(root, file))
 286
 287 def ensure_dir_exists(path):
 288         if not os.path.exists(path):
 289                 os.makedirs(path)
 290                 #shutil.rmtree(path)
 291         #os.makedirs(path)
 292
 293
 294 def path_leaf(path):
 295     head, tail = ntpath.split(path) #This is for windows?? Recheck
 296     return tail
 297
 298 def path_head(path):
 299     head, tail = ntpath.split(path)
 300     return head
 301
 302 def difflines(list1, list2):
 303     c = set(list1).union(set(list2))
 304     d = set(list1).intersection(set(list2))
 305     return list(c-d)
 306
 307 #Creating Diff between OLD and NEW attribute files v12
 308 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 309         if GenerateDiffAttr == "FALSE":
 310                 return
 311         with open(ATTR_OLD, 'r') as f_old:
 312                 lines1 = set(f_old.read().splitlines())
 313
 314         with open(ATTR_NEW, 'r') as f_new:
 315                 lines2 = set(f_new.read().splitlines())
 316
 317         lines = difflines(lines2, lines1)
 318         with open(ATTR_FILE, 'w+') as file_out:
 319                 for line in lines:
 320                         if line not in lines1:
 321                                 logging.info('Diff_AttrFiles - %s' % line)
 322                                 file_out.write(line+'\n')
 323
 324         f_new.close()
 325         f_old.close()
 326         file_out.close()
 327
 328
 329
 330 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 331         #Full File Path should MATCH
 332         if GenerateDiffAttr == "FALSE":
 333                 return
 334         FilePath = '"/'+RequestedPath+'"'
 335         #print ('FilePath - %s'% (FilePath))
 336         with open(AttributeFile) as f:
 337                 for line in f:
 338                         if FilePath in line:
 339                                 if Type == SYMLINK_TYPE:
 340                                         Sym_Attibutes.append(line)
 341                                 else:
 342                                         File_Attibutes.append(line)
 343
 344
 345 '''This function returns the SHA-1 hash of the file passed into it'''
 346 def hash_file(filename):
 347
 348    # make a hash object
 349    h = hashlib.sha1()
 350
 351    # open file for reading in binary mode
 352    with open(filename,'rb') as file:
 353        # loop till the end of the file
 354        chunk = 0
 355        while chunk != b'':
 356            # read only 1024 bytes at a time
 357            chunk = file.read(1024*1024)
 358            h.update(chunk)
 359
 360    # return the hex representation of digest
 361    return h.hexdigest()
 362
 363 def find_dupes_dir(BASE_OLD, BASE_NEW):
 364         dups = {}
 365         fdupes = {}
 366         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 367         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 368         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 369                 #print('Scanning %s...' % rootbase)
 370                 for filename in fileListB:
 371                         path = os.path.join(rootbase, filename)
 372                         if os.path.islink(path):
 373                                 continue
 374                         # Calculate hash
 375                         file_hash = hash_file(path)
 376                         dups[file_hash] = path
 377
 378         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 379                 #print('Scanning %s...' % roottarget)
 380                 for filename in fileListT:
 381                         # Get the path to the file
 382                         path = os.path.join(roottarget, filename)
 383                         if os.path.islink(path):
 384                                 continue
 385                         # Calculate hash
 386                         file_hash = hash_file(path)
 387                         # Add or append the file path
 388                         if file_hash in dups:
 389                                 BaseStr = dups.get(file_hash)
 390                                 Baseloc = path.find('/')
 391                                 TarLoc = BaseStr.find('/')
 392                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 393                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 394                                         fdupes[path] = BaseStr
 395         logging.info('Total Duplicate files %d' % (len(fdupes)))
 396         return fdupes
 397
 398
 399 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 400         dups = {}
 401         fdupes = {}
 402         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 403
 404         for filename in fileListB:
 405                 Src_File = BASE_OLD+'/'+filename
 406                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 407                         continue
 408                 # Calculate hash
 409                 file_hash = hash_file(Src_File)
 410                 dups[file_hash] = Src_File
 411
 412
 413         for filename in fileListT:
 414                 Dest_File = BASE_NEW+'/'+filename
 415                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 416                         continue
 417                 # Calculate hash
 418                 file_hash = hash_file(Dest_File)
 419                 if file_hash in dups:
 420                         BaseStr = dups.get(file_hash)
 421                         Baseloc = BaseStr.find('/')
 422                         if not BaseStr[Baseloc:] == filename:
 423                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 424                                 fdupes[BaseStr] = filename
 425
 426         logging.info('Total Duplicate files %d' % (len(fdupes)))
 427         return fdupes
 428
 429 def SS_UpdateSize(src_file, dst_file):
 430         global MEM_REQ
 431         global MEM_FILE
 432         oldsize_d= os.path.getsize(src_file)
 433         newsize_d= os.path.getsize(dst_file)
 434         if oldsize_d >= newsize_d:
 435                 Max = newsize_d
 436         else:
 437                 Max = oldsize_d
 438         if MEM_REQ < Max:
 439                 MEM_REQ = Max
 440                 MEM_FILE = dst_file
 441
 442
 443
 444 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 445         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 446         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 447
 448         # First let's fill up these categories
 449         files_new = []
 450         files_removed = []
 451         Dir_removed = []
 452         Dir_Added = []
 453         files_changed = []
 454         files_unchanged = []
 455         files_renamed = []
 456         File_Attibutes = []
 457         Sym_Attibutes = []
 458
 459         files_Del_List = {}
 460         files_New_List = {}
 461         MyDict_Patches = {}
 462
 463
 464
 465         PWD = os.getcwd()
 466
 467         # Generate NEW List
 468         for elt in New_files:
 469                 if elt not in Old_files:
 470                         files_new.append(elt)
 471                         logging.info('New files %s' % elt)
 472
 473         # Generate Delete List
 474         for elt in Old_files:
 475                 if elt not in New_files:
 476                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 477                         files_removed.append(elt)
 478                         logging.info('Old files %s' % elt)
 479
 480
 481         for elt in Old_dirs:
 482                 #print('List of Old Dirs %s' % elt)
 483                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 484                 if elt not in New_dirs:
 485                         Dir_removed.append(elt)
 486                         logging.info('Old Dirs %s' % elt+'/')
 487
 488         for elt in New_dirs:
 489                 if elt not in Old_dirs:
 490                         Dir_Added.append(elt)
 491                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 492
 493         # What files have changed contents but not name/path?
 494         for elt in New_files:
 495                 if elt in Old_files:
 496                         #Both are symbolic linkes and they differ
 497                         src_file = BASE_OLD+'/'+elt
 498                         dst_file = BASE_NEW+'/'+elt
 499                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 500                         if os.path.islink(src_file) and os.path.islink(dst_file):
 501                                 if not os.readlink(src_file) == os.readlink(dst_file):
 502                                         files_changed.append(elt)
 503                                         #print('%d Sym link files changed' % len(files_changed))
 504                                         logging.info('Sym links Changed - %s' % elt)
 505                                 else:
 506                                         files_unchanged.append(elt)
 507                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 508                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 509                                 if not filecmp.cmp(src_file, dst_file):
 510                                         files_changed.append(elt)
 511                                         #print('%d Normal files changed' % len(files_changed))
 512                                         #print('Files Changed - %s' % elt)
 513                                 else:
 514                                         files_unchanged.append(elt)
 515                         #File types differ between BASE and TARGET
 516                         else:
 517                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 518                                 #Both file types have changed and they differ
 519                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 520                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 521                                 files_removed.append(elt)
 522                                 files_new.append(elt)
 523
 524
 525         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 526         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 527
 528         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 529         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 530                 with open(VERBATIM_LIST, 'r') as F_News:
 531                         lines = set(F_News.read().splitlines())
 532                 for line in lines:
 533                         if line in files_changed:
 534                                 files_changed.remove(line)
 535                                 files_removed.append(line)
 536                         if line in files_new:
 537                                 files_new.remove(line)
 538
 539         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 540         if SUPPORT_RENAME == "TRUE":
 541                 for elt in files_removed:
 542                         if os.path.isfile(BASE_OLD+'/'+elt):
 543                                 FileName = path_leaf(elt)
 544                                 entries = re.split('[0-9]' , FileName)
 545                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 546                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 547                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 548                                 if len(entries[0]) > 0:
 549                                         files_Del_List.update({entries[0]: elt})
 550
 551                 for elt in files_new:
 552                         if os.path.isfile(BASE_NEW+'/'+elt):
 553                                 FileName = path_leaf(elt)
 554                                 entries = re.split('[0-9]' , FileName)
 555                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 556                                 if len(entries[0]) > 0:
 557                                         files_New_List.update({entries[0]: elt})
 558
 559                 for key, value in files_Del_List.iteritems():
 560                         #print('Key value pair -%s -%s' % (key, value))
 561                         if key in files_New_List:
 562                                 # this file is the same name in both!
 563                                 src_file = BASE_OLD+'/'+value
 564                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 565                                 olddirpath = path_head(files_New_List[key])
 566                                 newdirpath = path_head(value)
 567                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 568                                         logging.debug('Cannot diff as one of them is Symlink')
 569                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 570                                         logging.debug('Cannot diff as one of them is dir')
 571                                 else:
 572                                         #Pick the best diff of same type and diff names
 573                                         files_renamed.append([files_New_List[key], value])
 574                                         files_removed.remove(value)
 575                                         files_new.remove(files_New_List[key])
 576
 577         '''
 578         Patch Section
 579                 Partition.txt contains Protocol for UPI
 580                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 581         '''
 582         Sym_Diff_Cnt = 0
 583         Sym_New_Cnt = 0;
 584         Del_Cnt = 0
 585         New_Cnt = 0
 586         Diff_Cnt = 0
 587         Move_Cnt = 0
 588         Verbatim_Cnt = 0
 589         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 590         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 591         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 592
 593         print("writing diff'ed changed files...")
 594         for elt in files_changed:
 595                 dst_file = BASE_NEW+'/'+elt
 596                 src_file = BASE_OLD+'/'+elt
 597                 #Both files are symbolic links and they differ
 598                 if os.path.islink(dst_file) and os.path.islink(src_file):
 599                 #Both are symlinks and they differ
 600                         logging.debug(' File Changed is Link %s ' % dst_file)
 601                         patch = os.readlink(dst_file)
 602                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 603                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 604                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 605                 #Both are NORMAL files and they differ
 606                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 607                         #Both are files and they differ
 608                         Diff_Cnt = Diff_Cnt + 1
 609                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 610                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 611                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 612                         SS_UpdateSize(src_file, dst_file)
 613
 614                         FORMAT = "REG"
 615                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 616                         if ret is not 0:
 617                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 618                                 files_new.append(elt)
 619                                 Diff_Cnt = Diff_Cnt - 1
 620                         else:
 621                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 622
 623                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 624                 #Both differ but they are of diff types
 625                 else:
 626                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 627                         files_removed.append(elt)
 628                         files_new.append(elt)
 629
 630         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 631         for oldpath, newpath in fdupes.iteritems():
 632                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 633
 634         for elt in files_removed:
 635                 src_file = BASE_OLD+'/'+elt
 636                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 637                 if src_file in fdupes.keys():
 638                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 639                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 640                         Move_Cnt = Move_Cnt + 1
 641                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 642                         files_removed.remove(elt)
 643                         files_new.remove(fdupes[src_file])
 644
 645         #Should be placed after removing duplicates, else they will be filtered here.
 646         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 647         #       First Step: Sort & Filter out unwanted files
 648         # Minimum condition used is,
 649         #       1. File name should match 70%
 650         #       2. Extensions should be same
 651         #       3. File name length shd b greater than 3 char
 652         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 653         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 654
 655         Base_DelList = files_removed[:]
 656         Base_NewList = files_new[:]
 657         DelList = sorted(Base_DelList, key=path_leaf)
 658         NewList = sorted(Base_NewList, key=path_leaf)
 659         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 660
 661         Filter1 = []
 662         Filter2 = []
 663         #Remove unwanted items which we cant make diff with for rename logic
 664         for file in DelList:
 665                 if os.path.islink(BASE_OLD+'/'+file):
 666                         continue
 667                 elif os.path.isdir(BASE_OLD+'/'+file):
 668                         continue
 669                 else:
 670                         Filter1.append(file)
 671                         #logging.debug('Sorted del list - %s' % (file))
 672
 673         DelList = Filter1
 674
 675         for file in NewList:
 676                 if os.path.islink(BASE_NEW+'/'+file):
 677                         continue
 678                 elif os.path.isdir(BASE_NEW+'/'+file):
 679                         continue
 680                 elif len(path_leaf(file)) <= 3:
 681                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 682                         continue
 683                 else:
 684                         Filter2.append(file)
 685
 686         NewList = Filter2
 687
 688         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 689
 690         for new_file in NewList:
 691                 R_Flag = 'FALSE';
 692                 DirPathNew = path_head(new_file)
 693                 FileNameNew = path_leaf(new_file)
 694                 DiffSize = 0
 695                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 696                 New_fs = winning_patch_sz
 697                 winning_file = ''
 698
 699                 for del_file in DelList:
 700                         FileNameOld = path_leaf(del_file)
 701                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 702                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 703                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 704                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 705                                 v1 = abs(New_fs-Del_fs)
 706                                 v2 = (New_fs+Del_fs)/2
 707                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 708                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 709                                         continue
 710                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 711                                 R_Flag = 'TRUE';
 712                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 713                                 if (DiffSize < 0.8 * winning_patch_sz):
 714                                         winning_patch_sz = DiffSize
 715                                         winning_file = del_file
 716                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 717                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 718                                 break;
 719                 if len(winning_file) > 0:
 720                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 721                         files_renamed.append([new_file, winning_file])
 722                         DelList.remove(winning_file)
 723                         files_removed.remove(winning_file)
 724                         files_new.remove(new_file)
 725
 726         #********************** Files should NOT be deleted for any such renames ***********************
 727
 728         if SUPPORT_RENAME == "TRUE":
 729                 for elt in files_renamed:
 730                         src_file = BASE_OLD+'/'+elt[1]
 731                         dst_file = BASE_NEW+'/'+elt[0]
 732                         Diff_Cnt = Diff_Cnt + 1
 733                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 734                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 735                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 736                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 737                         # Should be careful of renaming files??
 738                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 739                         # What if folder path has numerics??
 740
 741                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 742                                 #This case never occurs??
 743                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 744                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 745                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 746                                 if filecmp.cmp(src_file, dst_file):
 747                                         Move_Cnt = Move_Cnt + 1
 748                                         Diff_Cnt = Diff_Cnt - 1
 749                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 750                                 else:
 751                                         FORMAT = "REG"
 752                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 753                                         if ret is not 0:
 754                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 755                                                 files_new.append(elt)
 756                                                 Diff_Cnt = Diff_Cnt - 1
 757                                         else:
 758                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 759
 760                                 SS_UpdateSize(src_file, dst_file)
 761                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 762
 763
 764         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 765         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 766
 767         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 768                 with open(VERBATIM_LIST, 'r') as F_News:
 769                         lines = set(F_News.read().splitlines())
 770                 for line in lines:
 771                         if not line in files_new:
 772                                 if os.path.exists(BASE_NEW+'/'+line):
 773                                         files_new.append(line)
 774                                         Verbatim_Cnt = Verbatim_Cnt+1
 775                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 776
 777
 778
 779         for elt in files_removed:
 780                 #if files are part of patches after renaming, we shd remove them as part of removed.
 781                 src_file = BASE_OLD+'/'+elt
 782                 if os.path.islink(src_file):
 783                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 784                 elif os.path.isdir(src_file):
 785                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 786                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 787                 else:
 788                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 789                 logging.debug(' File Deleted %s' % src_file)
 790                 Del_Cnt = Del_Cnt + 1
 791
 792         Dir_removed.sort(reverse=True)
 793         for elt in Dir_removed:
 794                 #if Dir is empty, add it to the removed list.
 795                 src_file = BASE_OLD+'/'+elt
 796                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 797                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 798                 if os.path.isdir(src_file):
 799                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 800                         Del_Cnt = Del_Cnt + 1
 801                         logging.debug(' Dir Deleted- %s' % src_file)
 802
 803
 804         for elt in files_new:
 805                 dst_file = BASE_NEW+'/'+elt
 806                 newfiles_dest_path = 'run/upgrade-sysroot/'
 807                 ensure_dir_exists(newfiles_dest_path)
 808                 if os.path.islink(dst_file):
 809                         patch = os.readlink(dst_file)
 810                         logging.debug(' File New Links %s' % elt)
 811                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 812                         #What if this is only a new sym link and folder already exists??? Should recheck
 813                         destpath = newfiles_dest_path + elt
 814                         if not os.path.exists(path_head(destpath)):
 815                                 os.makedirs(path_head(destpath))
 816                                 logging.info('New SymLink - Adding missing Dir')
 817                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 818                         Sym_New_Cnt = Sym_New_Cnt + 1
 819                 elif os.path.isdir(dst_file): # We create just empty directory here
 820                         destpath = newfiles_dest_path + elt
 821                         if not os.path.exists(destpath):
 822                                 os.makedirs(destpath)
 823                                 logging.debug(' File New Dir %s' % destpath)
 824                                 New_Cnt = New_Cnt + 1
 825                 else:
 826                         New_Cnt = New_Cnt + 1
 827                         destpath = newfiles_dest_path + elt
 828                         destdir = os.path.dirname(destpath)
 829                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 830
 831                         if not os.path.isdir(destdir):
 832                                 try:
 833                                         os.makedirs(destdir)
 834                                 except:
 835                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 836                                         raise
 837
 838                         try:
 839                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 840                                         shutil.copy2(dst_file, destpath)
 841                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 842                         except:
 843                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 844                                 raise
 845
 846         for elt in Dir_Added:
 847                 newfiles_dest_path = 'run/upgrade-sysroot/'
 848                 ensure_dir_exists(newfiles_dest_path)
 849                 destpath = newfiles_dest_path + elt
 850                 if not os.path.exists(destpath):
 851                         os.makedirs(destpath)
 852                         logging.debug(' DirList New Dir %s' % destpath)
 853                         New_Cnt = New_Cnt + 1
 854
 855         #Base directory should be system
 856         print 'Compressing New files'
 857         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 858                 WorkingDir = os.getcwd()
 859                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 860                 logging.info('Curr Working Dir - %s' % os.getcwd())
 861                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 862                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 863                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 864                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 865                 os.chdir(WorkingDir)
 866                 shutil.rmtree(NEW_FILES_PATH)
 867                 # use 7z a system.7z ./*
 868
 869         #logging.info('%d Dir to be removed' % len(Dir_removed))
 870         logging.info('%d files unchanged' % len(files_unchanged))
 871         logging.info('%d files files_renamed' % len(files_renamed))
 872         logging.info('%d files NEW' % len(files_new))
 873         logging.info('%d File attr' % len(File_Attibutes))
 874         logging.info('%d Sym attr' % len(Sym_Attibutes))
 875         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 876         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 877
 878         #There could be duplicates, TODO, can check before adding..
 879         ATTR_FILE_D = open(ATTR_FILE,'a+')
 880         for elt in File_Attibutes:
 881                 ATTR_FILE_D.write(elt)
 882         for elt in Sym_Attibutes:
 883                 ATTR_FILE_D.write(elt)
 884
 885         ATTR_FILE_D.close()
 886
 887         Partition_Doc_SymLinks.close()
 888         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 889         Partition_Doc.write(Partition_Read_SymLinks.read())
 890         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 891         Partition_Doc_SymLinks.close()
 892         Partition_Doc.close()
 893         os.remove(SymLinkDoc)
 894
 895         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 896                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 897                 logging.info('No Delta Generated for %s' % PART_NAME)
 898                 shutil.rmtree(OUT_DIR)
 899
 900
 901 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 902
 903         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 904         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 905         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 906         temp_apk = '../'+g_output_dir+'/'+b_apk
 907         Patch = 'Patch_'+b_apk
 908         ensure_dir_exists(Patch)
 909         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 910
 911         #Size issue on Device side?? shd check this
 912         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 913         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 914                 lines = set(f_new.read().splitlines())
 915                 for line in lines:
 916                         #print('Action ==> %s' % line)
 917                         #Action, Path, Patch = line.split('|')
 918                         Items = line.split('|')
 919                         Action = Items[0]
 920                         Path = Items[1]
 921                         ActualPath = a_folder+'/'+Path
 922                         PatchPath = Patch+'/'+Path
 923                         SrcPath = g_output_dir+'/'+path_leaf(Path)
 924                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 925                         if line[0] == 'c':
 926                                 patchName = g_output_dir+'/'+Items[2]
 927                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 928                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
 929                                 WorkingDir = os.getcwd()
 930                                 os.chdir(WorkingDir+"/"+"temp_a")
 931                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 932                                 os.chdir(WorkingDir)
 933                         elif line[0] == 's':
 934                                 WorkingDir = os.getcwd()
 935                                 os.chdir(WorkingDir+"/"+"temp_a")
 936                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 937                                 os.chdir(WorkingDir)
 938                         else:
 939                                 print('Apply_Container_Delta - Unknown Error')
 940         #print('Touch all files and set common attributes for DIFF generation')
 941         WorkingDir = os.getcwd()
 942         os.chdir(WorkingDir+"/"+Patch)
 943
 944         CONTAINER_DATE = '200011111111.11'
 945         CONTAINER_MODE = '0755'
 946         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
 947         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
 948         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
 949
 950         print 'Update Intermediate Archive'
 951         #subprocess.call(['zip','-ryX', b_apk, '*'])
 952         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
 953         os.chdir(WorkingDir)
 954         #print('Apply Path completed - Now create diff for this and place in patch folder')
 955         #print os.getcwd()
 956         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
 957         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
 958         patchLoc = '%s/%s' % (g_output_dir, patchName)
 959
 960         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
 961
 962         #Only on HOST... for testing
 963         if TEST_MODE == 'TRUE':
 964                 UpgradedName = '%s_Upgraded' % (b_apk)
 965                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
 966
 967         #This is file only with NEWS and empty diffs and same files.
 968         if TEST_MODE == 'FALSE':
 969                 os.remove(g_output_dir+'/'+b_apk)
 970                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
 971                 shutil.rmtree(Patch)
 972
 973 def IsSymlink(info):
 974   return (info.external_attr >> 16) == 0120777
 975
 976 def NewFiles(src, dest):
 977         print src,dest
 978         subprocess.call(['cp','-rp', src,dest])
 979     #try:
 980                 #shutil.copytree(src, dest)
 981     #except OSError as e:
 982         # If the error was caused because the source wasn't a directory
 983         #if e.errno == errno.ENOTDIR:
 984             #shutil.copy2(src, dest)
 985         #else:
 986             #print('Directory not copied. Error: %s' % e)
 987
 988 def measure_two_filediffs(src, dst):
 989         patchLoc = 'temp.patch'
 990         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
 991         result_size = os.path.getsize(patchLoc)
 992         os.remove(patchLoc)
 993         return result_size
 994
 995 def Get_Files(path):
 996         all_files = []
 997         all_dirs = []
 998
 999         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1000                 for directory in directories:
1001                         #DirName = os.path.join(root+'/',directory)
1002                         DirName = os.path.join(root,directory)
1003                         if os.path.islink(DirName):
1004                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1005                                 all_files.append(os.path.relpath(DirName, path))
1006                         elif not os.listdir(DirName):
1007                                 #print('*****Empty Directory******* -%s', DirName)
1008                                 #This should NOT be appended ??? Empty dir shd b considered
1009                                 all_dirs.append(os.path.relpath(DirName, path))
1010                         else:
1011                                 all_dirs.append(os.path.relpath(DirName, path))
1012                 for filename in filenames:
1013                         FileName = os.path.join(root,filename)
1014                         all_files.append(os.path.relpath(FileName, path))
1015
1016         all_files.sort()
1017         all_dirs.sort()
1018         return all_files, all_dirs
1019
1020
1021 USAGE_DOCSTRING = """
1022       Generate Delta using BASEOLD AND BASE NEW
1023           Attributes is optional
1024           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1025 """
1026
1027 def Usage(docstring):
1028   print docstring.rstrip("\n")
1029   print COMMON_DOCSTRING
1030
1031
1032
1033 if __name__ == '__main__':
1034         main()
1035