mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49 def global_paths():
  50         global DIFF_UTIL
  51         global ZIPUTIL
  52         global NEW_FILES_PATH
  53         global NEW_FILES_ZIP_NAME
  54         global SYMLINK_TYPE
  55         global ATTR_DOC_EXT
  56         global SYMLINK_DOC_NAME
  57         global DIFF_PREFIX
  58         global DIFF_SUFFIX
  59         global SUPPORT_RENAME
  60         global NEW_PREFIX
  61         global DIFFPATCH_UTIL
  62         global SUPPORT_CONTAINERS
  63         global FULL_IMAGE
  64         global DELTA_IMAGE
  65         global DELTA_FS
  66         global EXTRA
  67         global COMMON_BIN_PATH
  68         global MEM_REQ
  69         global EMPTY
  70         global VERBATIM_LIST
  71         global MEM_FILE
  72
  73 COMMON_BIN_PATH = "../../common/bin/"
  74 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  75 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  76 #ZIPUTIL = "p7zip "
  77 ZIPUTIL = "7z -mf=off a system.7z "
  78 NEW_FILES_PATH = "run/upgrade-sysroot"
  79 NEW_FILES_ZIP_NAME = "system.7z"
  80 SYMLINK_TYPE = "SYM"
  81 ATTR_DOC_EXT = "_attr.txt"
  82 SYMLINK_DOC_NAME = "_sym.txt"
  83 PART_DOC_EXT = ".txt"
  84 DIFF_PREFIX = "diff"
  85 DIFF_SUFFIX = ".delta"
  86 NEW_PREFIX = 'new'
  87 FULL_IMAGE = "FULL_IMAGE"
  88 DELTA_IMAGE = "DELTA_IMAGE"
  89 DELTA_FS = "DELTA_FS"
  90 EXTRA = "EXTRA"
  91 LOGFILE = "Delta.log"
  92 VERBATIM_LIST = "Verbatim_List.txt"
  93 EMPTY = ""
  94 MEM_REQ = 0
  95 MEM_FILE = "NULL"
  96 COMPRESSION_LZMA = "lzma"
  97 COMPRESSION_BROTLI = "brotli"
  98
  99 SUPPORT_RENAME = "TRUE" #Use appropriate name
 100 SUPPORT_CONTAINERS = "FALSE"
 101 SUPPORT_VERBATIM = "TRUE"
 102
 103 TEST_MODE = "FALSE"
 104
 105 def main():
 106         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 107         global AttributeFile
 108         global GenerateDiffAttr
 109         try:
 110
 111                 if len(sys.argv) < 5:
 112                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 113                 UPDATE_TYPE = sys.argv[1]
 114                 UPDATE_TYPE_S = UPDATE_TYPE.split(":")
 115                 PART_NAME = sys.argv[2]  # lets make this also optional
 116
 117                 BASE_OLD = sys.argv[3]
 118                 BASE_NEW = sys.argv[4]
 119                 OUT_DIR = sys.argv[5]
 120                 ATTR_OLD = EMPTY
 121                 ATTR_NEW = EMPTY
 122                 UPDATE_CFG_PATH = EMPTY
 123                 GenerateDiffAttr = "FALSE"
 124                 if UPDATE_TYPE_S[0] == DELTA_FS:
 125                         #instead of arguments check it in outdirectory ?
 126                         if len(sys.argv) == 9:
 127                                 ATTR_OLD = sys.argv[6]
 128                                 ATTR_NEW = sys.argv[7]
 129                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 130                                 GenerateDiffAttr = "TRUE"
 131
 132                 elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]:
 133                         if len(sys.argv) == 7:
 134                                 #Use path in better way
 135                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 136
 137                 global DIFF_UTIL
 138                 global DIFFPATCH_UTIL
 139                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 140                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 141                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 142                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 143                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 144                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 145                                 sys.exit(1)
 146
 147                 start = datetime.datetime.now().time()
 148                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 149                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 150
 151                 ensure_dir_exists(OUT_DIR)
 152                 if GenerateDiffAttr == "TRUE":
 153                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 154                                 print >> sys.stderr, "Attributes missing -- ABORT"
 155                                 sys.exit(1)
 156
 157
 158                 # Should check if APT is supported on other linux flavours
 159                 cache = apt.Cache()
 160                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 161                         logging.info ('Basic utils installed')
 162                 else:
 163                         print >> sys.stderr, "Basic utils missing -- ABORT"
 164                         sys.exit(1)
 165
 166                 if UPDATE_TYPE_S[0] == FULL_IMAGE:
 167                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 168                 # #### currently does not support LZMA ####
 169                 #  elif UPDATE_TYPE == DELTA_IMAGE:
 170                 #      SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
 171                 elif UPDATE_TYPE_S[0] == DELTA_IMAGE:
 172                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
 173                 elif UPDATE_TYPE == DELTA_FS:
 174                         AttributeFile = ATTR_NEW
 175                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 176                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 177                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 178                         New_files, New_dirs = Get_Files(BASE_NEW)
 179                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 180
 181                         if not UPDATE_CFG_PATH == EMPTY:
 182                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 183
 184
 185                 elif UPDATE_TYPE == EXTRA:
 186                         print('UPDATE_TYPE ---- EXTRA')
 187                 else:
 188                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 189
 190                 if GenerateDiffAttr == "TRUE":
 191                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 192                                 os.remove(ATTR_OLD)
 193                                 os.remove(ATTR_NEW)
 194                 end = datetime.datetime.now().time()
 195
 196                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 197                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 198                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 199                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 200
 201         except Exception as exc:
 202                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 203                 raise exc
 204
 205 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 206         f = open(UPDATE_CFG_PATH, 'r')
 207         lines = f.readlines()
 208         f.close()
 209         f = open(UPDATE_CFG_PATH, 'w')
 210         for line in lines:
 211                 ConfigItems = line.split()
 212                 if ConfigItems[0] == DELTA_BIN:
 213                         DELTA = ConfigItems[1]
 214                         logging.info ('Updating %s config' % DELTA_BIN)
 215                         line = line.rstrip('\n')
 216                         Value = MEM_REQ
 217                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 218                         f.write(line)
 219                 else:
 220                         f.write(line)
 221         f.close()
 222
 223 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
 224         #for sizes
 225
 226         oldsize_d= os.path.getsize(BASE_OLD)
 227         newsize_d= os.path.getsize(BASE_NEW)
 228         SHA_BIN_DEST= hash_file(BASE_NEW)
 229         SHA_BIN_BASE=hash_file(BASE_OLD)
 230
 231         #incase UPDATE CFG is empty
 232         DELTA = DELTA_BIN
 233         SS_UpdateSize(BASE_OLD, BASE_NEW)
 234         #Should throw error if PART NAME NOT found??
 235         if not UPDATE_CFG_PATH == EMPTY:
 236                 f = open(UPDATE_CFG_PATH, 'r')
 237                 lines = f.readlines()
 238                 f.close()
 239                 f = open(UPDATE_CFG_PATH, 'w')
 240                 for line in lines:
 241                         ConfigItems = line.split()
 242                         if ConfigItems[0] == DELTA_BIN:
 243                                 logging.info ('Updating %s config' % DELTA_BIN)
 244                                 DELTA = ConfigItems[1]
 245                                 line = line.rstrip('\n')
 246                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 247                                 f.write(line)
 248                         else:
 249                                 f.write(line)
 250                 f.close()
 251
 252         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 253         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 254         subprocess.call([DIFF_UTIL,"-c",COMPRESSION_METHOD,BASE_OLD,BASE_NEW,patchLoc])
 255
 256
 257
 258 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 259         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 260         oldsize_d= os.path.getsize(BASE_OLD)
 261         newsize_d= os.path.getsize(BASE_NEW)
 262         SHA_BIN_DEST= hash_file(BASE_NEW)
 263         SHA_BIN_BASE=hash_file(BASE_OLD)
 264         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 265         SS_UpdateSize(BASE_OLD, BASE_NEW)
 266
 267         if not UPDATE_CFG_PATH == EMPTY:
 268                 f = open(UPDATE_CFG_PATH, 'r')
 269                 lines = f.readlines()
 270                 f.close()
 271                 f = open(UPDATE_CFG_PATH, 'w')
 272                 for line in lines:
 273                         ConfigItems = line.split()
 274                         if ConfigItems[0] == DELTA_BIN:
 275                                 logging.info ('Updating %s config' % DELTA_BIN)
 276                                 DELTA = ConfigItems[1]
 277                                 line = line.rstrip('\n')
 278                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 279                                 f.write(line)
 280                         else:
 281                                 f.write(line)
 282                 f.close()
 283
 284 def zipdir(path, zip):
 285     for root, dirs, files in os.walk(path):
 286         for file in files:
 287             zip.write(os.path.join(root, file))
 288
 289 def ensure_dir_exists(path):
 290         if not os.path.exists(path):
 291                 os.makedirs(path)
 292                 #shutil.rmtree(path)
 293         #os.makedirs(path)
 294
 295
 296 def path_leaf(path):
 297     head, tail = ntpath.split(path) #This is for windows?? Recheck
 298     return tail
 299
 300 def path_head(path):
 301     head, tail = ntpath.split(path)
 302     return head
 303
 304
 305 #Creating Diff between OLD and NEW attribute files v12
 306 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 307         if GenerateDiffAttr == "FALSE":
 308                 return
 309         with open(ATTR_OLD, 'r') as f_old:
 310                 lines1 = set(f_old.read().splitlines())
 311
 312         with open(ATTR_NEW, 'r') as f_new:
 313                 lines2 = set(f_new.read().splitlines())
 314
 315         lines = set.difference(lines2, lines1)
 316         with open(ATTR_FILE, 'w+') as file_out:
 317                 for line in lines:
 318                         logging.info('Diff_AttrFiles - %s' % line)
 319                         file_out.write(line+'\n')
 320
 321
 322
 323 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 324         #Full File Path should MATCH
 325         if GenerateDiffAttr == "FALSE":
 326                 return
 327         FilePath = '"/'+RequestedPath+'"'
 328         #print ('FilePath - %s'% (FilePath))
 329         with open(AttributeFile) as f:
 330                 for line in f:
 331                         if FilePath in line:
 332                                 if Type == SYMLINK_TYPE:
 333                                         Sym_Attibutes.append(line)
 334                                 else:
 335                                         File_Attibutes.append(line)
 336
 337
 338 '''This function returns the SHA-1 hash of the file passed into it'''
 339 def hash_file(filename):
 340
 341    # make a hash object
 342    h = hashlib.sha1()
 343
 344    # open file for reading in binary mode
 345    with open(filename,'rb') as file:
 346        # loop till the end of the file
 347        chunk = 0
 348        while chunk != b'':
 349            # read only 1024 bytes at a time
 350            chunk = file.read(1024*1024)
 351            h.update(chunk)
 352
 353    # return the hex representation of digest
 354    return h.hexdigest()
 355
 356 def find_dupes_dir(BASE_OLD, BASE_NEW):
 357         dups = {}
 358         fdupes = {}
 359         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 360         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 361         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 362                 #print('Scanning %s...' % rootbase)
 363                 for filename in fileListB:
 364                         path = os.path.join(rootbase, filename)
 365                         if os.path.islink(path):
 366                                 continue
 367                         # Calculate hash
 368                         file_hash = hash_file(path)
 369                         dups[file_hash] = path
 370
 371         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 372                 #print('Scanning %s...' % roottarget)
 373                 for filename in fileListT:
 374                         # Get the path to the file
 375                         path = os.path.join(roottarget, filename)
 376                         if os.path.islink(path):
 377                                 continue
 378                         # Calculate hash
 379                         file_hash = hash_file(path)
 380                         # Add or append the file path
 381                         if file_hash in dups:
 382                                 BaseStr = dups.get(file_hash)
 383                                 Baseloc = path.find('/')
 384                                 TarLoc = BaseStr.find('/')
 385                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 386                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 387                                         fdupes[path] = BaseStr
 388         logging.info('Total Duplicate files %d' % (len(fdupes)))
 389         return fdupes
 390
 391
 392 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 393         dups = {}
 394         fdupes = {}
 395         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 396
 397         for filename in fileListB:
 398                 Src_File = BASE_OLD+'/'+filename
 399                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 400                         continue
 401                 # Calculate hash
 402                 file_hash = hash_file(Src_File)
 403                 dups[file_hash] = Src_File
 404
 405
 406         for filename in fileListT:
 407                 Dest_File = BASE_NEW+'/'+filename
 408                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 409                         continue
 410                 # Calculate hash
 411                 file_hash = hash_file(Dest_File)
 412                 if file_hash in dups:
 413                         BaseStr = dups.get(file_hash)
 414                         Baseloc = BaseStr.find('/')
 415                         if not BaseStr[Baseloc:] == filename:
 416                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 417                                 fdupes[BaseStr] = filename
 418
 419         logging.info('Total Duplicate files %d' % (len(fdupes)))
 420         return fdupes
 421
 422 def SS_UpdateSize(src_file, dst_file):
 423         global MEM_REQ
 424         global MEM_FILE
 425         oldsize_d= os.path.getsize(src_file)
 426         newsize_d= os.path.getsize(dst_file)
 427         if oldsize_d >= newsize_d:
 428                 Max = newsize_d
 429         else:
 430                 Max = oldsize_d
 431         if MEM_REQ < Max:
 432                 MEM_REQ = Max
 433                 MEM_FILE = dst_file
 434
 435
 436
 437 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 438         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 439         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 440
 441         # First let's fill up these categories
 442         files_new = []
 443         files_removed = []
 444         Dir_removed = []
 445         Dir_Added = []
 446         files_changed = []
 447         files_unchanged = []
 448         files_renamed = []
 449         File_Attibutes = []
 450         Sym_Attibutes = []
 451
 452         files_Del_List = {}
 453         files_New_List = {}
 454         MyDict_Patches = {}
 455
 456
 457
 458         PWD = os.getcwd()
 459
 460         # Generate NEW List
 461         for elt in New_files:
 462                 if elt not in Old_files:
 463                         files_new.append(elt)
 464                         logging.info('New files %s' % elt)
 465
 466         # Generate Delete List
 467         for elt in Old_files:
 468                 if elt not in New_files:
 469                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 470                         files_removed.append(elt)
 471                         logging.info('Old files %s' % elt)
 472
 473
 474         for elt in Old_dirs:
 475                 #print('List of Old Dirs %s' % elt)
 476                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 477                 if elt not in New_dirs:
 478                         Dir_removed.append(elt)
 479                         logging.info('Old Dirs %s' % elt+'/')
 480
 481         for elt in New_dirs:
 482                 if elt not in Old_dirs:
 483                         Dir_Added.append(elt)
 484                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 485
 486         # What files have changed contents but not name/path?
 487         for elt in New_files:
 488                 if elt in Old_files:
 489                         #Both are symbolic linkes and they differ
 490                         src_file = BASE_OLD+'/'+elt
 491                         dst_file = BASE_NEW+'/'+elt
 492                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 493                         if os.path.islink(src_file) and os.path.islink(dst_file):
 494                                 if not os.readlink(src_file) == os.readlink(dst_file):
 495                                         files_changed.append(elt)
 496                                         #print('%d Sym link files changed' % len(files_changed))
 497                                         logging.info('Sym links Changed - %s' % elt)
 498                                 else:
 499                                         files_unchanged.append(elt)
 500                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 501                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 502                                 if not filecmp.cmp(src_file, dst_file):
 503                                         files_changed.append(elt)
 504                                         #print('%d Normal files changed' % len(files_changed))
 505                                         #print('Files Changed - %s' % elt)
 506                                 else:
 507                                         files_unchanged.append(elt)
 508                         #File types differ between BASE and TARGET
 509                         else:
 510                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 511                                 #Both file types have changed and they differ
 512                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 513                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 514                                 files_removed.append(elt)
 515                                 files_new.append(elt)
 516
 517
 518         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 519         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 520
 521         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 522         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 523                 with open(VERBATIM_LIST, 'r') as F_News:
 524                         lines = set(F_News.read().splitlines())
 525                 for line in lines:
 526                         if line in files_changed:
 527                                 files_changed.remove(line)
 528                                 files_removed.append(line)
 529                         if line in files_new:
 530                                 files_new.remove(line)
 531
 532         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 533         if SUPPORT_RENAME == "TRUE":
 534                 for elt in files_removed:
 535                         if os.path.isfile(BASE_OLD+'/'+elt):
 536                                 FileName = path_leaf(elt)
 537                                 entries = re.split('[0-9]' , FileName)
 538                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 539                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 540                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 541                                 if len(entries[0]) > 0:
 542                                         files_Del_List.update({entries[0]: elt})
 543
 544                 for elt in files_new:
 545                         if os.path.isfile(BASE_NEW+'/'+elt):
 546                                 FileName = path_leaf(elt)
 547                                 entries = re.split('[0-9]' , FileName)
 548                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 549                                 if len(entries[0]) > 0:
 550                                         files_New_List.update({entries[0]: elt})
 551
 552                 for key, value in files_Del_List.iteritems():
 553                         #print('Key value pair -%s -%s' % (key, value))
 554                         if key in files_New_List:
 555                                 # this file is the same name in both!
 556                                 src_file = BASE_OLD+'/'+value
 557                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 558                                 olddirpath = path_head(files_New_List[key])
 559                                 newdirpath = path_head(value)
 560                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 561                                         logging.debug('Cannot diff as one of them is Symlink')
 562                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 563                                         logging.debug('Cannot diff as one of them is dir')
 564                                 else:
 565                                         #Pick the best diff of same type and diff names
 566                                         files_renamed.append([files_New_List[key], value])
 567                                         files_removed.remove(value)
 568                                         files_new.remove(files_New_List[key])
 569
 570         '''
 571         Patch Section
 572                 Partition.txt contains Protocol for UPI
 573                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 574         '''
 575         Sym_Diff_Cnt = 0
 576         Sym_New_Cnt = 0;
 577         Del_Cnt = 0
 578         New_Cnt = 0
 579         Diff_Cnt = 0
 580         Move_Cnt = 0
 581         Verbatim_Cnt = 0
 582         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 583         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 584         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 585
 586         print("writing diff'ed changed files...")
 587         for elt in files_changed:
 588                 dst_file = BASE_NEW+'/'+elt
 589                 src_file = BASE_OLD+'/'+elt
 590                 #Both files are symbolic links and they differ
 591                 if os.path.islink(dst_file) and os.path.islink(src_file):
 592                 #Both are symlinks and they differ
 593                         logging.debug(' File Changed is Link %s ' % dst_file)
 594                         patch = os.readlink(dst_file)
 595                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 596                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 597                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 598                 #Both are NORMAL files and they differ
 599                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 600                         #Both are files and they differ
 601                         Diff_Cnt = Diff_Cnt + 1
 602                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 603                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 604                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 605                         SS_UpdateSize(src_file, dst_file)
 606
 607                         FORMAT = "REG"
 608                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 609                         if ret is not 0:
 610                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 611                                 files_new.append(elt)
 612                                 Diff_Cnt = Diff_Cnt - 1
 613                         else:
 614                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 615
 616                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 617                 #Both differ but they are of diff types
 618                 else:
 619                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 620                         files_removed.append(elt)
 621                         files_new.append(elt)
 622
 623         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 624         for oldpath, newpath in fdupes.iteritems():
 625                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 626
 627         for elt in files_removed:
 628                 src_file = BASE_OLD+'/'+elt
 629                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 630                 if src_file in fdupes.keys():
 631                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 632                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 633                         Move_Cnt = Move_Cnt + 1
 634                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 635                         files_removed.remove(elt)
 636                         files_new.remove(fdupes[src_file])
 637
 638         #Should be placed after removing duplicates, else they will be filtered here.
 639         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 640         #       First Step: Sort & Filter out unwanted files
 641         # Minimum condition used is,
 642         #       1. File name should match 70%
 643         #       2. Extensions should be same
 644         #       3. File name length shd b greater than 3 char
 645         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 646         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 647
 648         Base_DelList = files_removed[:]
 649         Base_NewList = files_new[:]
 650         DelList = sorted(Base_DelList, key=path_leaf)
 651         NewList = sorted(Base_NewList, key=path_leaf)
 652         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 653
 654         Filter1 = []
 655         Filter2 = []
 656         #Remove unwanted items which we cant make diff with for rename logic
 657         for file in DelList:
 658                 if os.path.islink(BASE_OLD+'/'+file):
 659                         continue
 660                 elif os.path.isdir(BASE_OLD+'/'+file):
 661                         continue
 662                 else:
 663                         Filter1.append(file)
 664                         #logging.debug('Sorted del list - %s' % (file))
 665
 666         DelList = Filter1
 667
 668         for file in NewList:
 669                 if os.path.islink(BASE_NEW+'/'+file):
 670                         continue
 671                 elif os.path.isdir(BASE_NEW+'/'+file):
 672                         continue
 673                 elif len(path_leaf(file)) <= 3:
 674                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 675                         continue
 676                 else:
 677                         Filter2.append(file)
 678
 679         NewList = Filter2
 680
 681         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 682
 683         for new_file in NewList:
 684                 R_Flag = 'FALSE';
 685                 DirPathNew = path_head(new_file)
 686                 FileNameNew = path_leaf(new_file)
 687                 DiffSize = 0
 688                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 689                 New_fs = winning_patch_sz
 690                 winning_file = ''
 691
 692                 for del_file in DelList:
 693                         FileNameOld = path_leaf(del_file)
 694                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 695                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 696                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 697                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 698                                 v1 = abs(New_fs-Del_fs)
 699                                 v2 = (New_fs+Del_fs)/2
 700                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 701                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 702                                         continue
 703                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 704                                 R_Flag = 'TRUE';
 705                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 706                                 if (DiffSize < 0.8 * winning_patch_sz):
 707                                         winning_patch_sz = DiffSize
 708                                         winning_file = del_file
 709                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 710                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 711                                 break;
 712                 if len(winning_file) > 0:
 713                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 714                         files_renamed.append([new_file, winning_file])
 715                         DelList.remove(winning_file)
 716                         files_removed.remove(winning_file)
 717                         files_new.remove(new_file)
 718
 719         #********************** Files should NOT be deleted for any such renames ***********************
 720
 721         if SUPPORT_RENAME == "TRUE":
 722                 for elt in files_renamed:
 723                         src_file = BASE_OLD+'/'+elt[1]
 724                         dst_file = BASE_NEW+'/'+elt[0]
 725                         Diff_Cnt = Diff_Cnt + 1
 726                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 727                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 728                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 729                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 730                         # Should be careful of renaming files??
 731                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 732                         # What if folder path has numerics??
 733
 734                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 735                                 #This case never occurs??
 736                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 737                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 738                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 739                                 if filecmp.cmp(src_file, dst_file):
 740                                         Move_Cnt = Move_Cnt + 1
 741                                         Diff_Cnt = Diff_Cnt - 1
 742                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 743                                 else:
 744                                         FORMAT = "REG"
 745                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 746                                         if ret is not 0:
 747                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 748                                                 files_new.append(elt)
 749                                                 Diff_Cnt = Diff_Cnt - 1
 750                                         else:
 751                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 752
 753                                 SS_UpdateSize(src_file, dst_file)
 754                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 755
 756
 757         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 758         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 759
 760         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 761                 with open(VERBATIM_LIST, 'r') as F_News:
 762                         lines = set(F_News.read().splitlines())
 763                 for line in lines:
 764                         if not line in files_new:
 765                                 if os.path.exists(BASE_NEW+'/'+line):
 766                                         files_new.append(line)
 767                                         Verbatim_Cnt = Verbatim_Cnt+1
 768                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 769
 770
 771
 772         for elt in files_removed:
 773                 #if files are part of patches after renaming, we shd remove them as part of removed.
 774                 src_file = BASE_OLD+'/'+elt
 775                 if os.path.islink(src_file):
 776                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 777                 elif os.path.isdir(src_file):
 778                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 779                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 780                 else:
 781                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 782                 logging.debug(' File Deleted %s' % src_file)
 783                 Del_Cnt = Del_Cnt + 1
 784
 785         Dir_removed.sort(reverse=True)
 786         for elt in Dir_removed:
 787                 #if Dir is empty, add it to the removed list.
 788                 src_file = BASE_OLD+'/'+elt
 789                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 790                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 791                 if os.path.isdir(src_file):
 792                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 793                         Del_Cnt = Del_Cnt + 1
 794                         logging.debug(' Dir Deleted- %s' % src_file)
 795
 796
 797         for elt in files_new:
 798                 dst_file = BASE_NEW+'/'+elt
 799                 newfiles_dest_path = 'run/upgrade-sysroot/'
 800                 ensure_dir_exists(newfiles_dest_path)
 801                 if os.path.islink(dst_file):
 802                         patch = os.readlink(dst_file)
 803                         logging.debug(' File New Links %s' % elt)
 804                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 805                         #What if this is only a new sym link and folder already exists??? Should recheck
 806                         destpath = newfiles_dest_path + elt
 807                         if not os.path.exists(path_head(destpath)):
 808                                 os.makedirs(path_head(destpath))
 809                                 logging.info('New SymLink - Adding missing Dir')
 810                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 811                         Sym_New_Cnt = Sym_New_Cnt + 1
 812                 elif os.path.isdir(dst_file): # We create just empty directory here
 813                         destpath = newfiles_dest_path + elt
 814                         if not os.path.exists(destpath):
 815                                 os.makedirs(destpath)
 816                                 logging.debug(' File New Dir %s' % destpath)
 817                                 New_Cnt = New_Cnt + 1
 818                 else:
 819                         New_Cnt = New_Cnt + 1
 820                         destpath = newfiles_dest_path + elt
 821                         destdir = os.path.dirname(destpath)
 822                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 823
 824                         if not os.path.isdir(destdir):
 825                                 try:
 826                                         os.makedirs(destdir)
 827                                 except Exception as exc:
 828                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 829                                         raise exc
 830
 831                         try:
 832                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 833                                         shutil.copy2(dst_file, destpath)
 834                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 835                         except Exception as exc:
 836                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 837                                 raise exc
 838
 839         for elt in Dir_Added:
 840                 newfiles_dest_path = 'run/upgrade-sysroot/'
 841                 ensure_dir_exists(newfiles_dest_path)
 842                 destpath = newfiles_dest_path + elt
 843                 if not os.path.exists(destpath):
 844                         os.makedirs(destpath)
 845                         logging.debug(' DirList New Dir %s' % destpath)
 846                         New_Cnt = New_Cnt + 1
 847
 848         #Base directory should be system
 849         print 'Compressing New files'
 850         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 851                 WorkingDir = os.getcwd()
 852                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 853                 logging.info('Curr Working Dir - %s' % os.getcwd())
 854                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 855                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 856                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 857                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 858                 os.chdir(WorkingDir)
 859                 shutil.rmtree(NEW_FILES_PATH)
 860                 # use 7z a system.7z ./*
 861
 862         #logging.info('%d Dir to be removed' % len(Dir_removed))
 863         logging.info('%d files unchanged' % len(files_unchanged))
 864         logging.info('%d files files_renamed' % len(files_renamed))
 865         logging.info('%d files NEW' % len(files_new))
 866         logging.info('%d File attr' % len(File_Attibutes))
 867         logging.info('%d Sym attr' % len(Sym_Attibutes))
 868         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 869         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 870
 871         #There could be duplicates, TODO, can check before adding..
 872         ATTR_FILE_D = open(ATTR_FILE,'a+')
 873         for elt in File_Attibutes:
 874                 ATTR_FILE_D.write(elt)
 875         for elt in Sym_Attibutes:
 876                 ATTR_FILE_D.write(elt)
 877
 878         ATTR_FILE_D.close()
 879
 880         Partition_Doc_SymLinks.close()
 881         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 882         Partition_Doc.write(Partition_Read_SymLinks.read())
 883         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 884         Partition_Doc_SymLinks.close()
 885         Partition_Doc.close()
 886         os.remove(SymLinkDoc)
 887
 888         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 889                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 890                 logging.info('No Delta Generated for %s' % PART_NAME)
 891                 shutil.rmtree(OUT_DIR)
 892
 893
 894 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 895
 896         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 897         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 898         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 899         temp_apk = '../'+g_output_dir+'/'+b_apk
 900         Patch = 'Patch_'+b_apk
 901         ensure_dir_exists(Patch)
 902         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 903
 904         #Size issue on Device side?? shd check this
 905         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 906         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 907                 lines = set(f_new.read().splitlines())
 908                 for line in lines:
 909                         #print('Action ==> %s' % line)
 910                         #Action, Path, Patch = line.split('|')
 911                         Items = line.split('|')
 912                         Action = Items[0]
 913                         Path = Items[1]
 914                         ActualPath = a_folder+'/'+Path
 915                         PatchPath = Patch+'/'+Path
 916                         SrcPath = g_output_dir+'/'+path_leaf(Path)
 917                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 918                         if line[0] == 'c':
 919                                 patchName = g_output_dir+'/'+Items[2]
 920                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 921                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
 922                                 WorkingDir = os.getcwd()
 923                                 os.chdir(WorkingDir+"/"+"temp_a")
 924                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 925                                 os.chdir(WorkingDir)
 926                         elif line[0] == 's':
 927                                 WorkingDir = os.getcwd()
 928                                 os.chdir(WorkingDir+"/"+"temp_a")
 929                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 930                                 os.chdir(WorkingDir)
 931                         else:
 932                                 print('Apply_Container_Delta - Unknown Error')
 933         #print('Touch all files and set common attributes for DIFF generation')
 934         WorkingDir = os.getcwd()
 935         os.chdir(WorkingDir+"/"+Patch)
 936
 937         CONTAINER_DATE = '200011111111.11'
 938         CONTAINER_MODE = '0755'
 939         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
 940         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
 941         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
 942
 943         print 'Update Intermediate Archive'
 944         #subprocess.call(['zip','-ryX', b_apk, '*'])
 945         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
 946         os.chdir(WorkingDir)
 947         #print('Apply Path completed - Now create diff for this and place in patch folder')
 948         #print os.getcwd()
 949         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
 950         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
 951         patchLoc = '%s/%s' % (g_output_dir, patchName)
 952
 953         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
 954
 955         #Only on HOST... for testing
 956         if TEST_MODE == 'TRUE':
 957                 UpgradedName = '%s_Upgraded' % (b_apk)
 958                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
 959
 960         #This is file only with NEWS and empty diffs and same files.
 961         if TEST_MODE == 'FALSE':
 962                 os.remove(g_output_dir+'/'+b_apk)
 963                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
 964                 shutil.rmtree(Patch)
 965
 966 def IsSymlink(info):
 967   return (info.external_attr >> 16) == 0120777
 968
 969 def NewFiles(src, dest):
 970         print src,dest
 971         subprocess.call(['cp','-rp', src,dest])
 972     #try:
 973                 #shutil.copytree(src, dest)
 974     #except OSError as e:
 975         # If the error was caused because the source wasn't a directory
 976         #if e.errno == errno.ENOTDIR:
 977             #shutil.copy2(src, dest)
 978         #else:
 979             #print('Directory not copied. Error: %s' % e)
 980
 981 def measure_two_filediffs(src, dst):
 982         patchLoc = 'temp.patch'
 983         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
 984         result_size = os.path.getsize(patchLoc)
 985         os.remove(patchLoc)
 986         return result_size
 987
 988 def Get_Files(path):
 989         all_files = []
 990         all_dirs = []
 991
 992         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
 993                 for directory in directories:
 994                         #DirName = os.path.join(root+'/',directory)
 995                         DirName = os.path.join(root,directory)
 996                         if os.path.islink(DirName):
 997                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
 998                                 all_files.append(os.path.relpath(DirName, path))
 999                         elif not os.listdir(DirName):
1000                                 #print('*****Empty Directory******* -%s', DirName)
1001                                 #This should NOT be appended ??? Empty dir shd b considered
1002                                 all_dirs.append(os.path.relpath(DirName, path))
1003                         else:
1004                                 all_dirs.append(os.path.relpath(DirName, path))
1005                 for filename in filenames:
1006                         FileName = os.path.join(root,filename)
1007                         all_files.append(os.path.relpath(FileName, path))
1008
1009         all_files.sort()
1010         all_dirs.sort()
1011         return all_files, all_dirs
1012
1013
1014 USAGE_DOCSTRING = """
1015       Generate Delta using BASEOLD AND BASE NEW
1016           Attributes is optional
1017           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1018 """
1019
1020 def Usage(docstring):
1021   print docstring.rstrip("\n")
1022   print COMMON_DOCSTRING
1023
1024
1025
1026 if __name__ == '__main__':
1027         main()
1028