mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49
  50 def global_paths():
  51         global DIFF_UTIL
  52         global ZIPUTIL
  53         global NEW_FILES_PATH
  54         global NEW_FILES_ZIP_NAME
  55         global SYMLINK_TYPE
  56         global ATTR_DOC_EXT
  57         global SYMLINK_DOC_NAME
  58         global DIFF_PREFIX
  59         global DIFF_SUFFIX
  60         global SUPPORT_RENAME
  61         global NEW_PREFIX
  62         global DIFFPATCH_UTIL
  63         global SUPPORT_CONTAINERS
  64         global FULL_IMAGE
  65         global DELTA_IMAGE
  66         global DELTA_FS
  67         global EXTRA
  68         global COMMON_BIN_PATH
  69         global MEM_REQ
  70         global EMPTY
  71         global VERBATIM_LIST
  72         global MEM_FILE
  73
  74
  75 COMMON_BIN_PATH = "../../common/bin/"
  76 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  77 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  78 #ZIPUTIL = "p7zip "
  79 ZIPUTIL = "7z -mf=off a system.7z "
  80 NEW_FILES_PATH = "run/upgrade-sysroot"
  81 NEW_FILES_ZIP_NAME = "system.7z"
  82 SYMLINK_TYPE = "SYM"
  83 ATTR_DOC_EXT = "_attr.txt"
  84 SYMLINK_DOC_NAME = "_sym.txt"
  85 PART_DOC_EXT = ".txt"
  86 DIFF_PREFIX = "diff"
  87 DIFF_SUFFIX = ".delta"
  88 NEW_PREFIX = 'new'
  89 FULL_IMAGE = "FULL_IMAGE"
  90 DELTA_IMAGE = "DELTA_IMAGE"
  91 DELTA_FS = "DELTA_FS"
  92 EXTRA = "EXTRA"
  93 LOGFILE = "Delta.log"
  94 VERBATIM_LIST = "Verbatim_List.txt"
  95 EMPTY = ""
  96 MEM_REQ = 0
  97 MEM_FILE = "NULL"
  98 COMPRESSION_LZMA = "lzma"
  99 COMPRESSION_BROTLI = "brotli"
 100
 101 SUPPORT_RENAME = "TRUE"  # Use appropriate name
 102 SUPPORT_CONTAINERS = "FALSE"
 103 SUPPORT_VERBATIM = "TRUE"
 104
 105 TEST_MODE = "FALSE"
 106
 107
 108 def main():
 109         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 110         global AttributeFile
 111         global GenerateDiffAttr
 112         try:
 113
 114                 if len(sys.argv) < 5:
 115                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 116                 UPDATE_TYPE = sys.argv[1]
 117                 UPDATE_TYPE_S = UPDATE_TYPE.split(":")
 118                 PART_NAME = sys.argv[2]  # lets make this also optional
 119
 120                 BASE_OLD = sys.argv[3]
 121                 BASE_NEW = sys.argv[4]
 122                 OUT_DIR = sys.argv[5]
 123                 ATTR_OLD = EMPTY
 124                 ATTR_NEW = EMPTY
 125                 UPDATE_CFG_PATH = EMPTY
 126                 GenerateDiffAttr = "FALSE"
 127                 if UPDATE_TYPE_S[0] == DELTA_FS:
 128                         #instead of arguments check it in outdirectory ?
 129                         if len(sys.argv) == 9:
 130                                 ATTR_OLD = sys.argv[6]
 131                                 ATTR_NEW = sys.argv[7]
 132                                 UPDATE_CFG_PATH = '../' + sys.argv[8]
 133                                 GenerateDiffAttr = "TRUE"
 134
 135                 elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]:
 136                         if len(sys.argv) == 7:
 137                                 #Use path in better way
 138                                 UPDATE_CFG_PATH = '../' + sys.argv[6]
 139
 140                 global DIFF_UTIL
 141                 global DIFFPATCH_UTIL
 142                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 143                         DIFF_UTIL = COMMON_BIN_PATH + DIFF_UTIL
 144                         DIFFPATCH_UTIL = COMMON_BIN_PATH + DIFFPATCH_UTIL
 145                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 146                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 147                                 logging.info('Diff Util Does NOT exist -- ABORT')
 148                                 sys.exit(1)
 149
 150                 start = datetime.datetime.now().time()
 151                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 152                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]' % (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 153
 154                 ensure_dir_exists(OUT_DIR)
 155                 if GenerateDiffAttr == "TRUE":
 156                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 157                                 print >> sys.stderr, "Attributes missing -- ABORT"
 158                                 sys.exit(1)
 159
 160                 # Should check if APT is supported on other linux flavours
 161                 cache = apt.Cache()
 162                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 163                         logging.info('Basic utils installed')
 164                 else:
 165                         print >> sys.stderr, "Basic utils missing -- ABORT"
 166                         sys.exit(1)
 167
 168                 if UPDATE_TYPE_S[0] == FULL_IMAGE:
 169                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 170                 # #### currently does not support LZMA ####
 171                 #  elif UPDATE_TYPE == DELTA_IMAGE:
 172                 #      SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
 173                 elif UPDATE_TYPE_S[0] == DELTA_IMAGE:
 174                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
 175                 elif UPDATE_TYPE == DELTA_FS:
 176                         AttributeFile = ATTR_NEW
 177                         ATTR_FILE = OUT_DIR + '/' + PART_NAME + ATTR_DOC_EXT
 178                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 179                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 180                         New_files, New_dirs = Get_Files(BASE_NEW)
 181                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 182
 183                         if not UPDATE_CFG_PATH == EMPTY:
 184                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 185
 186                 elif UPDATE_TYPE == EXTRA:
 187                         print('UPDATE_TYPE ---- EXTRA')
 188                 else:
 189                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 190
 191                 if GenerateDiffAttr == "TRUE":
 192                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 193                                 os.remove(ATTR_OLD)
 194                                 os.remove(ATTR_NEW)
 195                 end = datetime.datetime.now().time()
 196
 197                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 198                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 199                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 200                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 201
 202         except Exception as exc:
 203                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 204                 raise exc
 205
 206
 207 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 208         f = open(UPDATE_CFG_PATH, 'r')
 209         lines = f.readlines()
 210         f.close()
 211         f = open(UPDATE_CFG_PATH, 'w')
 212         for line in lines:
 213                 ConfigItems = line.split()
 214                 if ConfigItems[0] == DELTA_BIN:
 215                         DELTA = ConfigItems[1]
 216                         logging.info('Updating %s config' % DELTA_BIN)
 217                         line = line.rstrip('\n')
 218                         Value = MEM_REQ
 219                         line = line.replace(line, line + '\t' + str(Value) + '\n')
 220                         f.write(line)
 221                 else:
 222                         f.write(line)
 223         f.close()
 224
 225
 226 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
 227         #for sizes
 228
 229         oldsize_d = os.path.getsize(BASE_OLD)
 230         newsize_d = os.path.getsize(BASE_NEW)
 231         SHA_BIN_DEST = hash_file(BASE_NEW)
 232         SHA_BIN_BASE = hash_file(BASE_OLD)
 233
 234         #incase UPDATE CFG is empty
 235         DELTA = DELTA_BIN
 236         SS_UpdateSize(BASE_OLD, BASE_NEW)
 237         #Should throw error if PART NAME NOT found??
 238         if not UPDATE_CFG_PATH == EMPTY:
 239                 f = open(UPDATE_CFG_PATH, 'r')
 240                 lines = f.readlines()
 241                 f.close()
 242                 f = open(UPDATE_CFG_PATH, 'w')
 243                 for line in lines:
 244                         ConfigItems = line.split()
 245                         if ConfigItems[0] == DELTA_BIN:
 246                                 logging.info('Updating %s config' % DELTA_BIN)
 247                                 DELTA = ConfigItems[1]
 248                                 line = line.rstrip('\n')
 249                                 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
 250                                 f.write(line)
 251                         else:
 252                                 f.write(line)
 253                 f.close()
 254
 255         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 256         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW, DELTA_BIN, patchLoc))
 257         subprocess.call([DIFF_UTIL, "-c", COMPRESSION_METHOD, BASE_OLD, BASE_NEW, patchLoc])
 258
 259
 260 def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 261         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW, DELTA_BIN))
 262         oldsize_d = os.path.getsize(BASE_OLD)
 263         newsize_d = os.path.getsize(BASE_NEW)
 264         SHA_BIN_DEST = hash_file(BASE_NEW)
 265         SHA_BIN_BASE = hash_file(BASE_OLD)
 266         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 267         SS_UpdateSize(BASE_OLD, BASE_NEW)
 268
 269         if not UPDATE_CFG_PATH == EMPTY:
 270                 f = open(UPDATE_CFG_PATH, 'r')
 271                 lines = f.readlines()
 272                 f.close()
 273                 f = open(UPDATE_CFG_PATH, 'w')
 274                 for line in lines:
 275                         ConfigItems = line.split()
 276                         if ConfigItems[0] == DELTA_BIN:
 277                                 logging.info('Updating %s config' % DELTA_BIN)
 278                                 DELTA = ConfigItems[1]
 279                                 line = line.rstrip('\n')
 280                                 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
 281                                 f.write(line)
 282                         else:
 283                                 f.write(line)
 284                 f.close()
 285
 286
 287 def zipdir(path, zip):
 288         for root, dirs, files in os.walk(path):
 289                 for file in files:
 290                         zip.write(os.path.join(root, file))
 291
 292
 293 def ensure_dir_exists(path):
 294         if not os.path.exists(path):
 295                 os.makedirs(path)
 296                 #shutil.rmtree(path)
 297         #os.makedirs(path)
 298
 299
 300 def path_leaf(path):
 301         head, tail = ntpath.split(path)  # This is for windows?? Recheck
 302         return tail
 303
 304
 305 def path_head(path):
 306         head, tail = ntpath.split(path)
 307         return head
 308
 309
 310 # Creating Diff between OLD and NEW attribute files v12
 311 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 312         if GenerateDiffAttr == "FALSE":
 313                 return
 314         with open(ATTR_OLD, 'r') as f_old:
 315                 lines1 = set(f_old.read().splitlines())
 316
 317         with open(ATTR_NEW, 'r') as f_new:
 318                 lines2 = set(f_new.read().splitlines())
 319
 320         lines = set.difference(lines2, lines1)
 321         with open(ATTR_FILE, 'w+') as file_out:
 322                 for line in lines:
 323                         logging.info('Diff_AttrFiles - %s' % line)
 324                         file_out.write(line + '\n')
 325
 326
 327 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 328         # Full File Path should MATCH
 329         if GenerateDiffAttr == "FALSE":
 330                 return
 331         FilePath = '"/' + RequestedPath + '"'
 332         #print ('FilePath - %s'% (FilePath))
 333         with open(AttributeFile) as f:
 334                 for line in f:
 335                         if FilePath in line:
 336                                 if Type == SYMLINK_TYPE:
 337                                         Sym_Attibutes.append(line)
 338                                 else:
 339                                         File_Attibutes.append(line)
 340
 341
 342 def hash_file(filename):
 343         '''This function returns the SHA-1 hash of the file passed into it'''
 344
 345         # make a hash object
 346         h = hashlib.sha1()
 347
 348         # open file for reading in binary mode
 349         with open(filename, 'rb') as file:
 350                 # loop till the end of the file
 351                 chunk = 0
 352                 while chunk != b'':
 353                         # read only 1024 bytes at a time
 354                         chunk = file.read(1024 * 1024)
 355                         h.update(chunk)
 356
 357         # return the hex representation of digest
 358         return h.hexdigest()
 359
 360
 361 def find_dupes_dir(BASE_OLD, BASE_NEW):
 362         dups = {}
 363         fdupes = {}
 364         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 365         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 366         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 367                 #print('Scanning %s...' % rootbase)
 368                 for filename in fileListB:
 369                         path = os.path.join(rootbase, filename)
 370                         if os.path.islink(path):
 371                                 continue
 372                         # Calculate hash
 373                         file_hash = hash_file(path)
 374                         dups[file_hash] = path
 375
 376         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 377                 #print('Scanning %s...' % roottarget)
 378                 for filename in fileListT:
 379                         # Get the path to the file
 380                         path = os.path.join(roottarget, filename)
 381                         if os.path.islink(path):
 382                                 continue
 383                         # Calculate hash
 384                         file_hash = hash_file(path)
 385                         # Add or append the file path
 386                         if file_hash in dups:
 387                                 BaseStr = dups.get(file_hash)
 388                                 Baseloc = path.find('/')
 389                                 TarLoc = BaseStr.find('/')
 390                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 391                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 392                                         fdupes[path] = BaseStr
 393         logging.info('Total Duplicate files %d' % (len(fdupes)))
 394         return fdupes
 395
 396
 397 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 398         dups = {}
 399         fdupes = {}
 400         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 401
 402         for filename in fileListB:
 403                 Src_File = BASE_OLD + '/' + filename
 404                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 405                         continue
 406                 # Calculate hash
 407                 file_hash = hash_file(Src_File)
 408                 dups[file_hash] = Src_File
 409
 410         for filename in fileListT:
 411                 Dest_File = BASE_NEW + '/' + filename
 412                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 413                         continue
 414                 # Calculate hash
 415                 file_hash = hash_file(Dest_File)
 416                 if file_hash in dups:
 417                         BaseStr = dups.get(file_hash)
 418                         Baseloc = BaseStr.find('/')
 419                         if not BaseStr[Baseloc:] == filename:
 420                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 421                                 fdupes[BaseStr] = filename
 422
 423         logging.info('Total Duplicate files %d' % (len(fdupes)))
 424         return fdupes
 425
 426
 427 def SS_UpdateSize(src_file, dst_file):
 428         global MEM_REQ
 429         global MEM_FILE
 430         oldsize_d = os.path.getsize(src_file)
 431         newsize_d = os.path.getsize(dst_file)
 432         if oldsize_d >= newsize_d:
 433                 Max = newsize_d
 434         else:
 435                 Max = oldsize_d
 436         if MEM_REQ < Max:
 437                 MEM_REQ = Max
 438                 MEM_FILE = dst_file
 439
 440
 441 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 442         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 443         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 444
 445         # First let's fill up these categories
 446         files_new = []
 447         files_removed = []
 448         Dir_removed = []
 449         Dir_Added = []
 450         files_changed = []
 451         files_unchanged = []
 452         files_renamed = []
 453         File_Attibutes = []
 454         Sym_Attibutes = []
 455
 456         files_Del_List = {}
 457         files_New_List = {}
 458         MyDict_Patches = {}
 459
 460         PWD = os.getcwd()
 461
 462         # Generate NEW List
 463         for elt in New_files:
 464                 if elt not in Old_files:
 465                         files_new.append(elt)
 466                         logging.info('New files %s' % elt)
 467
 468         # Generate Delete List
 469         for elt in Old_files:
 470                 if elt not in New_files:
 471                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 472                         files_removed.append(elt)
 473                         logging.info('Old files %s' % elt)
 474
 475         for elt in Old_dirs:
 476                 #print('List of Old Dirs %s' % elt)
 477                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 478                 if elt not in New_dirs:
 479                         Dir_removed.append(elt)
 480                         logging.info('Old Dirs %s' % elt + '/')
 481
 482         for elt in New_dirs:
 483                 if elt not in Old_dirs:
 484                         Dir_Added.append(elt)
 485                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 486
 487         # What files have changed contents but not name/path?
 488         for elt in New_files:
 489                 if elt in Old_files:
 490                         # Both are symbolic linkes and they differ
 491                         src_file = BASE_OLD + '/' + elt
 492                         dst_file = BASE_NEW + '/' + elt
 493                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 494                         if os.path.islink(src_file) and os.path.islink(dst_file):
 495                                 if not os.readlink(src_file) == os.readlink(dst_file):
 496                                         files_changed.append(elt)
 497                                         #print('%d Sym link files changed' % len(files_changed))
 498                                         logging.info('Sym links Changed - %s' % elt)
 499                                 else:
 500                                         files_unchanged.append(elt)
 501                         # Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 502                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 503                                 if not filecmp.cmp(src_file, dst_file):
 504                                         files_changed.append(elt)
 505                                         #print('%d Normal files changed' % len(files_changed))
 506                                         #print('Files Changed - %s' % elt)
 507                                 else:
 508                                         files_unchanged.append(elt)
 509                         # File types differ between BASE and TARGET
 510                         else:
 511                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 512                                 # Both file types have changed and they differ
 513                                 # Case 1: First Delete the OLD entry file type (Be it anything)
 514                                 # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 515                                 files_removed.append(elt)
 516                                 files_new.append(elt)
 517
 518         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 519         # This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 520
 521         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 522         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 523                 with open(VERBATIM_LIST, 'r') as F_News:
 524                         lines = set(F_News.read().splitlines())
 525                 for line in lines:
 526                         if line in files_changed:
 527                                 files_changed.remove(line)
 528                                 files_removed.append(line)
 529                         if line in files_new:
 530                                 files_new.remove(line)
 531
 532         # Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 533         if SUPPORT_RENAME == "TRUE":
 534                 for elt in files_removed:
 535                         if os.path.isfile(BASE_OLD + '/' + elt):
 536                                 FileName = path_leaf(elt)
 537                                 entries = re.split('[0-9]', FileName)
 538                                 # Gives the STRING part of NAME. if name starts with version then later part wil b string
 539                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 540                                 # If version is starting at the begining of the string?? shd we hav additional check for such cases??
 541                                 if len(entries[0]) > 0:
 542                                         files_Del_List.update({entries[0]: elt})
 543
 544                 for elt in files_new:
 545                         if os.path.isfile(BASE_NEW + '/' + elt):
 546                                 FileName = path_leaf(elt)
 547                                 entries = re.split('[0-9]', FileName)
 548                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 549                                 if len(entries[0]) > 0:
 550                                         files_New_List.update({entries[0]: elt})
 551
 552                 for key, value in files_Del_List.iteritems():
 553                         #print('Key value pair -%s -%s' % (key, value))
 554                         if key in files_New_List:
 555                                 # this file is the same name in both!
 556                                 src_file = BASE_OLD + '/' + value
 557                                 dst_file = BASE_NEW + '/' + files_New_List[key]
 558                                 olddirpath = path_head(files_New_List[key])
 559                                 newdirpath = path_head(value)
 560                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 561                                         logging.debug('Cannot diff as one of them is Symlink')
 562                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 563                                         logging.debug('Cannot diff as one of them is dir')
 564                                 else:
 565                                         #Pick the best diff of same type and diff names
 566                                         files_renamed.append([files_New_List[key], value])
 567                                         files_removed.remove(value)
 568                                         files_new.remove(files_New_List[key])
 569
 570         '''
 571         Patch Section
 572                 Partition.txt contains Protocol for UPI
 573                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 574         '''
 575         Sym_Diff_Cnt = 0
 576         Sym_New_Cnt = 0
 577         Del_Cnt = 0
 578         New_Cnt = 0
 579         Diff_Cnt = 0
 580         Move_Cnt = 0
 581         Verbatim_Cnt = 0
 582         SymLinkDoc = OUT_DIR + '/' + PART_NAME + SYMLINK_DOC_NAME
 583         Partition_Doc = open(OUT_DIR + '/' + PART_NAME + '.txt', 'w')
 584         Partition_Doc_SymLinks = open(SymLinkDoc, 'w')
 585
 586         print("writing diff'ed changed files...")
 587         for elt in files_changed:
 588                 dst_file = BASE_NEW + '/' + elt
 589                 src_file = BASE_OLD + '/' + elt
 590                 # Both files are symbolic links and they differ
 591                 if os.path.islink(dst_file) and os.path.islink(src_file):
 592                         # Both are symlinks and they differ
 593                         logging.debug(' File Changed is Link %s ' % dst_file)
 594                         patch = os.readlink(dst_file)
 595                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 596                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 597                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 598                 # Both are NORMAL files and they differ
 599                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 600                         # Both are files and they differ
 601                         Diff_Cnt = Diff_Cnt + 1
 602                         patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 603                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 604                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 605                         SS_UpdateSize(src_file, dst_file)
 606
 607                         FORMAT = "REG"
 608                         ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
 609                         if ret is not 0:
 610                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 611                                 files_new.append(elt)
 612                                 Diff_Cnt = Diff_Cnt - 1
 613                         else:
 614                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 615
 616                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 617                 # Both differ but they are of diff types
 618                 else:
 619                         # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 620                         files_removed.append(elt)
 621                         files_new.append(elt)
 622
 623         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 624         for oldpath, newpath in fdupes.iteritems():
 625                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 626
 627         for elt in files_removed:
 628                 src_file = BASE_OLD + '/' + elt
 629                 # If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 630                 if src_file in fdupes.keys():
 631                         dst_file = BASE_NEW + '/' + fdupes[src_file]
 632                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 633                         Move_Cnt = Move_Cnt + 1
 634                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 635                         files_removed.remove(elt)
 636                         files_new.remove(fdupes[src_file])
 637
 638         # Should be placed after removing duplicates, else they will be filtered here.
 639         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 640         # First Step: Sort & Filter out unwanted files
 641         # Minimum condition used is,
 642         #       1. File name should match 70%
 643         #       2. Extensions should be same
 644         #       3. File name length shd b greater than 3 char
 645         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 646         #       5. Should consider editdistance for RENAME LOGIC ==> TBD
 647
 648         Base_DelList = files_removed[:]
 649         Base_NewList = files_new[:]
 650         DelList = sorted(Base_DelList, key=path_leaf)
 651         NewList = sorted(Base_NewList, key=path_leaf)
 652         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 653
 654         Filter1 = []
 655         Filter2 = []
 656         # Remove unwanted items which we cant make diff with for rename logic
 657         for file in DelList:
 658                 if os.path.islink(BASE_OLD + '/' + file):
 659                         continue
 660                 elif os.path.isdir(BASE_OLD + '/' + file):
 661                         continue
 662                 else:
 663                         Filter1.append(file)
 664                         #logging.debug('Sorted del list - %s' % (file))
 665
 666         DelList = Filter1
 667
 668         for file in NewList:
 669                 if os.path.islink(BASE_NEW + '/' + file):
 670                         continue
 671                 elif os.path.isdir(BASE_NEW + '/' + file):
 672                         continue
 673                 elif len(path_leaf(file)) <= 3:
 674                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW + '/' + file))
 675                         continue
 676                 else:
 677                         Filter2.append(file)
 678
 679         NewList = Filter2
 680
 681         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 682
 683         for new_file in NewList:
 684                 R_Flag = 'FALSE'
 685                 DirPathNew = path_head(new_file)
 686                 FileNameNew = path_leaf(new_file)
 687                 DiffSize = 0
 688                 winning_patch_sz = os.path.getsize(BASE_NEW + '/' + new_file)
 689                 New_fs = winning_patch_sz
 690                 winning_file = ''
 691
 692                 for del_file in DelList:
 693                         FileNameOld = path_leaf(del_file)
 694                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 695                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 696                                 # Percentage difference between two file sizes is within 30%, then we consider for diff generation
 697                                 Del_fs = os.path.getsize(BASE_OLD + '/' + del_file)
 698                                 v1 = abs(New_fs - Del_fs)
 699                                 v2 = (New_fs + Del_fs) / 2
 700                                 if(v2 <= 0 or ((v1 / v2) * 100) > 30):
 701                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 702                                         continue
 703                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 704                                 R_Flag = 'TRUE'
 705                                 DiffSize = measure_two_filediffs(BASE_OLD + '/' + del_file, BASE_NEW + '/' + new_file)
 706                                 if (DiffSize < 0.8 * winning_patch_sz):
 707                                         winning_patch_sz = DiffSize
 708                                         winning_file = del_file
 709                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and R_Flag == 'TRUE'):
 710                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 711                                 break
 712                 if len(winning_file) > 0:
 713                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 714                         files_renamed.append([new_file, winning_file])
 715                         DelList.remove(winning_file)
 716                         files_removed.remove(winning_file)
 717                         files_new.remove(new_file)
 718
 719         #********************** Files should NOT be deleted for any such renames ***********************
 720
 721         if SUPPORT_RENAME == "TRUE":
 722                 for elt in files_renamed:
 723                         src_file = BASE_OLD + '/' + elt[1]
 724                         dst_file = BASE_NEW + '/' + elt[0]
 725                         Diff_Cnt = Diff_Cnt + 1
 726                         patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 727                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 728                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 729                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 730                         # Should be careful of renaming files??
 731                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 732                         # What if folder path has numerics??
 733
 734                         if os.path.isdir(src_file) or os.path.isdir(dst_file):
 735                                 # This case never occurs??
 736                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 737                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 738                         else:  # Make sure these files are PROPER and they shd NOT be symlinks
 739                                 if filecmp.cmp(src_file, dst_file):
 740                                         Move_Cnt = Move_Cnt + 1
 741                                         Diff_Cnt = Diff_Cnt - 1
 742                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 743                                 else:
 744                                         FORMAT = "REG"
 745                                         ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
 746                                         if ret is not 0:
 747                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 748                                                 files_new.append(elt)
 749                                                 Diff_Cnt = Diff_Cnt - 1
 750                                         else:
 751                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 752
 753                                 SS_UpdateSize(src_file, dst_file)
 754                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 755
 756         # HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 757         # So that, the rename functionality will NOT create PATCH instead of verbatims.
 758
 759         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 760                 with open(VERBATIM_LIST, 'r') as F_News:
 761                         lines = set(F_News.read().splitlines())
 762                 for line in lines:
 763                         if line not in files_new:
 764                                 if os.path.exists(BASE_NEW + '/' + line):
 765                                         files_new.append(line)
 766                                         Verbatim_Cnt = Verbatim_Cnt + 1
 767                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW + '/' + line)
 768
 769         for elt in files_removed:
 770                 # if files are part of patches after renaming, we shd remove them as part of removed.
 771                 src_file = BASE_OLD + '/' + elt
 772                 if os.path.islink(src_file):
 773                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 774                 elif os.path.isdir(src_file):
 775                         # If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 776                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 777                 else:
 778                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 779                 logging.debug(' File Deleted %s' % src_file)
 780                 Del_Cnt = Del_Cnt + 1
 781
 782         Dir_removed.sort(reverse=True)
 783         for elt in Dir_removed:
 784                 # if Dir is empty, add it to the removed list.
 785                 src_file = BASE_OLD + '/' + elt
 786                 # Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 787                 # It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 788                 if os.path.isdir(src_file):
 789                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 790                         Del_Cnt = Del_Cnt + 1
 791                         logging.debug(' Dir Deleted- %s' % src_file)
 792
 793         for elt in files_new:
 794                 dst_file = BASE_NEW + '/' + elt
 795                 newfiles_dest_path = 'run/upgrade-sysroot/'
 796                 ensure_dir_exists(newfiles_dest_path)
 797                 if os.path.islink(dst_file):
 798                         patch = os.readlink(dst_file)
 799                         logging.debug(' File New Links %s' % elt)
 800                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 801                         # What if this is only a new sym link and folder already exists??? Should recheck
 802                         destpath = newfiles_dest_path + elt
 803                         if not os.path.exists(path_head(destpath)):
 804                                 os.makedirs(path_head(destpath))
 805                                 logging.info('New SymLink - Adding missing Dir')
 806                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 807                         Sym_New_Cnt = Sym_New_Cnt + 1
 808                 elif os.path.isdir(dst_file):  # We create just empty directory here
 809                         destpath = newfiles_dest_path + elt
 810                         if not os.path.exists(destpath):
 811                                 os.makedirs(destpath)
 812                                 logging.debug(' File New Dir %s' % destpath)
 813                                 New_Cnt = New_Cnt + 1
 814                 else:
 815                         New_Cnt = New_Cnt + 1
 816                         destpath = newfiles_dest_path + elt
 817                         destdir = os.path.dirname(destpath)
 818                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 819
 820                         if not os.path.isdir(destdir):
 821                                 try:
 822                                         os.makedirs(destdir)
 823                                 except Exception as exc:
 824                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 825                                         raise exc
 826
 827                         try:
 828                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 829                                         shutil.copy2(dst_file, destpath)
 830                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 831                         except Exception as exc:
 832                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 833                                 raise exc
 834
 835         for elt in Dir_Added:
 836                 newfiles_dest_path = 'run/upgrade-sysroot/'
 837                 ensure_dir_exists(newfiles_dest_path)
 838                 destpath = newfiles_dest_path + elt
 839                 if not os.path.exists(destpath):
 840                         os.makedirs(destpath)
 841                         logging.debug(' DirList New Dir %s' % destpath)
 842                         New_Cnt = New_Cnt + 1
 843
 844         # Base directory should be system
 845         print 'Compressing New files'
 846         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 847                 WorkingDir = os.getcwd()
 848                 os.chdir(os.getcwd() + "/" + NEW_FILES_PATH)
 849                 logging.info('Curr Working Dir - %s' % os.getcwd())
 850                 os.system(ZIPUTIL + NEW_FILES_PATH + " >> " + LOGFILE)
 851                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR)
 852                 # New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 853                 SS_UpdateSize(WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME)
 854                 os.chdir(WorkingDir)
 855                 shutil.rmtree(NEW_FILES_PATH)
 856                 # use 7z a system.7z ./*
 857
 858         #logging.info('%d Dir to be removed' % len(Dir_removed))
 859         logging.info('%d files unchanged' % len(files_unchanged))
 860         logging.info('%d files files_renamed' % len(files_renamed))
 861         logging.info('%d files NEW' % len(files_new))
 862         logging.info('%d File attr' % len(File_Attibutes))
 863         logging.info('%d Sym attr' % len(Sym_Attibutes))
 864         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 865         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 866
 867         # There could be duplicates, TODO, can check before adding..
 868         ATTR_FILE_D = open(ATTR_FILE, 'a+')
 869         for elt in File_Attibutes:
 870                 ATTR_FILE_D.write(elt)
 871         for elt in Sym_Attibutes:
 872                 ATTR_FILE_D.write(elt)
 873
 874         ATTR_FILE_D.close()
 875
 876         Partition_Doc_SymLinks.close()
 877         Partition_Read_SymLinks = open(SymLinkDoc, 'r+')
 878         Partition_Doc.write(Partition_Read_SymLinks.read())
 879         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 880         Partition_Doc_SymLinks.close()
 881         Partition_Doc.close()
 882         os.remove(SymLinkDoc)
 883
 884         if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 885                 print('No Delta Generated for %s - %s' % (PART_NAME, OUT_DIR))
 886                 logging.info('No Delta Generated for %s' % PART_NAME)
 887                 shutil.rmtree(OUT_DIR)
 888
 889
 890 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 891
 892         # CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 893         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 894         shutil.copy2(g_output_dir + '/' + b_apk, g_output_dir + '/temp')
 895         temp_apk = '../' + g_output_dir + '/' + b_apk
 896         Patch = 'Patch_' + b_apk
 897         ensure_dir_exists(Patch)
 898         shutil.copy2(g_output_dir + '/' + b_apk, Patch + '/' + b_apk)
 899
 900         # Size issue on Device side?? shd check this
 901         subprocess.call(['unzip', '-q', Patch + '/' + b_apk, '-d', Patch])
 902         with open(g_output_dir + '/PATCH.txt', 'r') as f_new:
 903                 lines = set(f_new.read().splitlines())
 904                 for line in lines:
 905                         #print('Action ==> %s' % line)
 906                         #Action, Path, Patch = line.split('|')
 907                         Items = line.split('|')
 908                         Action = Items[0]
 909                         Path = Items[1]
 910                         ActualPath = a_folder + '/' + Path
 911                         PatchPath = Patch + '/' + Path
 912                         SrcPath = g_output_dir + '/' + path_leaf(Path)
 913                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 914                         if line[0] == 'c':
 915                                 patchName = g_output_dir + '/' + Items[2]
 916                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 917                                 subprocess.call([DIFFPATCH_UTIL, ActualPath, ActualPath, patchName])
 918                                 WorkingDir = os.getcwd()
 919                                 os.chdir(WorkingDir + "/" + "temp_a")
 920                                 subprocess.call(['cp', '--parents', Path, '../' + Patch])
 921                                 os.chdir(WorkingDir)
 922                         elif line[0] == 's':
 923                                 WorkingDir = os.getcwd()
 924                                 os.chdir(WorkingDir + "/" + "temp_a")
 925                                 subprocess.call(['cp', '--parents', Path, '../' + Patch])
 926                                 os.chdir(WorkingDir)
 927                         else:
 928                                 print('Apply_Container_Delta - Unknown Error')
 929         #print('Touch all files and set common attributes for DIFF generation')
 930         WorkingDir = os.getcwd()
 931         os.chdir(WorkingDir + "/" + Patch)
 932
 933         CONTAINER_DATE = '200011111111.11'
 934         CONTAINER_MODE = '0755'
 935         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
 936         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
 937         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../' + Patch])
 938
 939         print 'Update Intermediate Archive'
 940         #subprocess.call(['zip','-ryX', b_apk, '*'])
 941         subprocess.call(['zip', '-ryX', b_apk] + glob.glob('*'))
 942         os.chdir(WorkingDir)
 943         #print('Apply Path completed - Now create diff for this and place in patch folder')
 944         #print os.getcwd()
 945         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir + '/' + b_apk, new_apk))
 946         patchName = ('New' + '_%s' + DIFF_SUFFIX) % (b_apk)
 947         patchLoc = '%s/%s' % (g_output_dir, patchName)
 948
 949         subprocess.call([DIFF_UTIL, Patch + '/' + b_apk, new_apk, patchLoc])
 950
 951         # Only on HOST... for testing
 952         if TEST_MODE == 'TRUE':
 953                 UpgradedName = '%s_Upgraded' % (b_apk)
 954                 subprocess.call([DIFFPATCH_UTIL, Patch + '/' + b_apk, UpgradedName, patchLoc])
 955
 956         # This is file only with NEWS and empty diffs and same files.
 957         if TEST_MODE == 'FALSE':
 958                 os.remove(g_output_dir + '/' + b_apk)
 959                 os.rename(g_output_dir + '/temp', g_output_dir + '/' + b_apk)
 960                 shutil.rmtree(Patch)
 961
 962
 963 def IsSymlink(info):
 964         return (info.external_attr >> 16) == 0120777
 965
 966
 967 def NewFiles(src, dest):
 968         print src, dest
 969         subprocess.call(['cp', '-rp', src, dest])
 970         #try:
 971                 #shutil.copytree(src, dest)
 972         #except OSError as e:
 973                 # If the error was caused because the source wasn't a directory
 974                 #if e.errno == errno.ENOTDIR:
 975                         #shutil.copy2(src, dest)
 976                 #else:
 977                         #print('Directory not copied. Error: %s' % e)
 978
 979
 980 def measure_two_filediffs(src, dst):
 981         patchLoc = 'temp.patch'
 982         subprocess.call([DIFF_UTIL, src, dst, patchLoc])
 983         result_size = os.path.getsize(patchLoc)
 984         os.remove(patchLoc)
 985         return result_size
 986
 987
 988 def Get_Files(path):
 989         all_files = []
 990         all_dirs = []
 991
 992         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
 993                 for directory in directories:
 994                         #DirName = os.path.join(root+'/',directory)
 995                         DirName = os.path.join(root, directory)
 996                         if os.path.islink(DirName):
 997                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
 998                                 all_files.append(os.path.relpath(DirName, path))
 999                         elif not os.listdir(DirName):
1000                                 #print('*****Empty Directory******* -%s', DirName)
1001                                 # This should NOT be appended ??? Empty dir shd b considered
1002                                 all_dirs.append(os.path.relpath(DirName, path))
1003                         else:
1004                                 all_dirs.append(os.path.relpath(DirName, path))
1005                 for filename in filenames:
1006                         FileName = os.path.join(root, filename)
1007                         all_files.append(os.path.relpath(FileName, path))
1008
1009         all_files.sort()
1010         all_dirs.sort()
1011         return all_files, all_dirs
1012
1013
1014 USAGE_DOCSTRING = """
1015         Generate Delta using BASEOLD AND BASE NEW
1016         Attributes is optional
1017         Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1018 """
1019
1020
1021 def Usage(docstring):
1022         print docstring.rstrip("\n")
1023         print COMMON_DOCSTRING
1024
1025
1026 if __name__ == '__main__':
1027         main()