mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49 def global_paths():
  50         global DIFF_UTIL
  51         global ZIPUTIL
  52         global NEW_FILES_PATH
  53         global NEW_FILES_ZIP_NAME
  54         global SYMLINK_TYPE
  55         global ATTR_DOC_EXT
  56         global SYMLINK_DOC_NAME
  57         global DIFF_PREFIX
  58         global DIFF_SUFFIX
  59         global SUPPORT_RENAME
  60         global NEW_PREFIX
  61         global DIFFPATCH_UTIL
  62         global SUPPORT_CONTAINERS
  63         global FULL_IMAGE
  64         global DELTA_IMAGE
  65         global DELTA_FS
  66         global EXTRA
  67         global COMMON_BIN_PATH
  68         global MEM_REQ
  69         global EMPTY
  70         global VERBATIM_LIST
  71         global MEM_FILE
  72
  73 COMMON_BIN_PATH = "../../common/bin/"
  74 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  75 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  76 #ZIPUTIL = "p7zip "
  77 ZIPUTIL = "7z -mf=off a system.7z "
  78 NEW_FILES_PATH = "run/upgrade-sysroot"
  79 NEW_FILES_ZIP_NAME = "system.7z"
  80 SYMLINK_TYPE = "SYM"
  81 ATTR_DOC_EXT = "_attr.txt"
  82 SYMLINK_DOC_NAME = "_sym.txt"
  83 PART_DOC_EXT = ".txt"
  84 DIFF_PREFIX = "diff"
  85 DIFF_SUFFIX = ".delta"
  86 NEW_PREFIX = 'new'
  87 FULL_IMAGE = "FULL_IMAGE"
  88 DELTA_IMAGE = "DELTA_IMAGE"
  89 DELTA_FS = "DELTA_FS"
  90 EXTRA = "EXTRA"
  91 LOGFILE = "Delta.log"
  92 VERBATIM_LIST = "Verbatim_List.txt"
  93 EMPTY = ""
  94 MEM_REQ = 0
  95 MEM_FILE = "NULL"
  96 COMPRESSION_LZMA = "lzma"
  97 COMPRESSION_BROTLI = "brotli"
  98
  99 SUPPORT_RENAME = "TRUE" #Use appropriate name
 100 SUPPORT_CONTAINERS = "FALSE"
 101 SUPPORT_VERBATIM = "TRUE"
 102
 103 TEST_MODE = "FALSE"
 104
 105 def main():
 106         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 107         global AttributeFile
 108         global GenerateDiffAttr
 109         try:
 110
 111                 if len(sys.argv) < 5:
 112                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 113                 UPDATE_TYPE = sys.argv[1]
 114                 UPDATE_TYPE_S = UPDATE_TYPE.split(":")
 115                 PART_NAME = sys.argv[2]  # lets make this also optional
 116
 117                 BASE_OLD = sys.argv[3]
 118                 BASE_NEW = sys.argv[4]
 119                 OUT_DIR = sys.argv[5]
 120                 ATTR_OLD = EMPTY
 121                 ATTR_NEW = EMPTY
 122                 UPDATE_CFG_PATH = EMPTY
 123                 GenerateDiffAttr = "FALSE"
 124                 if UPDATE_TYPE_S[0] == DELTA_FS:
 125                         #instead of arguments check it in outdirectory ?
 126                         if len(sys.argv) == 9:
 127                                 ATTR_OLD = sys.argv[6]
 128                                 ATTR_NEW = sys.argv[7]
 129                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 130                                 GenerateDiffAttr = "TRUE"
 131
 132                 elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]:
 133                         if len(sys.argv) == 7:
 134                                 #Use path in better way
 135                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 136
 137                 global DIFF_UTIL
 138                 global DIFFPATCH_UTIL
 139                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 140                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 141                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 142                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 143                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 144                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 145                                 sys.exit(1)
 146
 147                 start = datetime.datetime.now().time()
 148                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 149                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 150
 151                 ensure_dir_exists(OUT_DIR)
 152                 if GenerateDiffAttr == "TRUE":
 153                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 154                                 print >> sys.stderr, "Attributes missing -- ABORT"
 155                                 sys.exit(1)
 156
 157
 158                 # Should check if APT is supported on other linux flavours
 159                 cache = apt.Cache()
 160                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 161                         logging.info ('Basic utils installed')
 162                 else:
 163                         print >> sys.stderr, "Basic utils missing -- ABORT"
 164                         sys.exit(1)
 165
 166                 if UPDATE_TYPE_S[0] == FULL_IMAGE:
 167                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 168                 # #### currently does not support LZMA ####
 169                 #  elif UPDATE_TYPE == DELTA_IMAGE:
 170                 #      SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
 171                 elif UPDATE_TYPE_S[0] == DELTA_IMAGE:
 172                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
 173                 elif UPDATE_TYPE == DELTA_FS:
 174                         AttributeFile = ATTR_NEW
 175                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 176                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 177                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 178                         New_files, New_dirs = Get_Files(BASE_NEW)
 179                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 180
 181                         if not UPDATE_CFG_PATH == EMPTY:
 182                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 183
 184
 185                 elif UPDATE_TYPE == EXTRA:
 186                         print('UPDATE_TYPE ---- EXTRA')
 187                 else:
 188                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 189
 190                 if GenerateDiffAttr == "TRUE":
 191                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 192                                 os.remove(ATTR_OLD)
 193                                 os.remove(ATTR_NEW)
 194                 end = datetime.datetime.now().time()
 195
 196                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 197                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 198                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 199                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 200
 201         except:
 202                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 203                 raise
 204
 205 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 206         f = open(UPDATE_CFG_PATH, 'r')
 207         lines = f.readlines()
 208         f.close()
 209         f = open(UPDATE_CFG_PATH, 'w')
 210         for line in lines:
 211                 ConfigItems = line.split()
 212                 if ConfigItems[0] == DELTA_BIN:
 213                         DELTA = ConfigItems[1]
 214                         logging.info ('Updating %s config' % DELTA_BIN)
 215                         line = line.rstrip('\n')
 216                         Value = MEM_REQ
 217                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 218                         f.write(line)
 219                 else:
 220                         f.write(line)
 221         f.close()
 222
 223 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
 224         #for sizes
 225
 226         oldsize_d= os.path.getsize(BASE_OLD)
 227         newsize_d= os.path.getsize(BASE_NEW)
 228         SHA_BIN_DEST= hash_file(BASE_NEW)
 229         SHA_BIN_BASE=hash_file(BASE_OLD)
 230
 231         #incase UPDATE CFG is empty
 232         DELTA = DELTA_BIN
 233         SS_UpdateSize(BASE_OLD, BASE_NEW)
 234         #Should throw error if PART NAME NOT found??
 235         if not UPDATE_CFG_PATH == EMPTY:
 236                 f = open(UPDATE_CFG_PATH, 'r')
 237                 lines = f.readlines()
 238                 f.close()
 239                 f = open(UPDATE_CFG_PATH, 'w')
 240                 for line in lines:
 241                         ConfigItems = line.split()
 242                         if ConfigItems[0] == DELTA_BIN:
 243                                 logging.info ('Updating %s config' % DELTA_BIN)
 244                                 DELTA = ConfigItems[1]
 245                                 line = line.rstrip('\n')
 246                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 247                                 f.write(line)
 248                         else:
 249                                 f.write(line)
 250                 f.close()
 251
 252         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 253         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 254         subprocess.call([DIFF_UTIL,"-c",COMPRESSION_METHOD,BASE_OLD,BASE_NEW,patchLoc])
 255
 256
 257
 258 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 259         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 260         oldsize_d= os.path.getsize(BASE_OLD)
 261         newsize_d= os.path.getsize(BASE_NEW)
 262         SHA_BIN_DEST= hash_file(BASE_NEW)
 263         SHA_BIN_BASE=hash_file(BASE_OLD)
 264         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 265         SS_UpdateSize(BASE_OLD, BASE_NEW)
 266
 267         if not UPDATE_CFG_PATH == EMPTY:
 268                 f = open(UPDATE_CFG_PATH, 'r')
 269                 lines = f.readlines()
 270                 f.close()
 271                 f = open(UPDATE_CFG_PATH, 'w')
 272                 for line in lines:
 273                         ConfigItems = line.split()
 274                         if ConfigItems[0] == DELTA_BIN:
 275                                 logging.info ('Updating %s config' % DELTA_BIN)
 276                                 DELTA = ConfigItems[1]
 277                                 line = line.rstrip('\n')
 278                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 279                                 f.write(line)
 280                         else:
 281                                 f.write(line)
 282                 f.close()
 283
 284 def zipdir(path, zip):
 285     for root, dirs, files in os.walk(path):
 286         for file in files:
 287             zip.write(os.path.join(root, file))
 288
 289 def ensure_dir_exists(path):
 290         if not os.path.exists(path):
 291                 os.makedirs(path)
 292                 #shutil.rmtree(path)
 293         #os.makedirs(path)
 294
 295
 296 def path_leaf(path):
 297     head, tail = ntpath.split(path) #This is for windows?? Recheck
 298     return tail
 299
 300 def path_head(path):
 301     head, tail = ntpath.split(path)
 302     return head
 303
 304 def difflines(list1, list2):
 305     c = set(list1).union(set(list2))
 306     d = set(list1).intersection(set(list2))
 307     return list(c-d)
 308
 309 #Creating Diff between OLD and NEW attribute files v12
 310 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 311         if GenerateDiffAttr == "FALSE":
 312                 return
 313         with open(ATTR_OLD, 'r') as f_old:
 314                 lines1 = set(f_old.read().splitlines())
 315
 316         with open(ATTR_NEW, 'r') as f_new:
 317                 lines2 = set(f_new.read().splitlines())
 318
 319         lines = difflines(lines2, lines1)
 320         with open(ATTR_FILE, 'w+') as file_out:
 321                 for line in lines:
 322                         if line not in lines1:
 323                                 logging.info('Diff_AttrFiles - %s' % line)
 324                                 file_out.write(line+'\n')
 325
 326         f_new.close()
 327         f_old.close()
 328         file_out.close()
 329
 330
 331
 332 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 333         #Full File Path should MATCH
 334         if GenerateDiffAttr == "FALSE":
 335                 return
 336         FilePath = '"/'+RequestedPath+'"'
 337         #print ('FilePath - %s'% (FilePath))
 338         with open(AttributeFile) as f:
 339                 for line in f:
 340                         if FilePath in line:
 341                                 if Type == SYMLINK_TYPE:
 342                                         Sym_Attibutes.append(line)
 343                                 else:
 344                                         File_Attibutes.append(line)
 345
 346
 347 '''This function returns the SHA-1 hash of the file passed into it'''
 348 def hash_file(filename):
 349
 350    # make a hash object
 351    h = hashlib.sha1()
 352
 353    # open file for reading in binary mode
 354    with open(filename,'rb') as file:
 355        # loop till the end of the file
 356        chunk = 0
 357        while chunk != b'':
 358            # read only 1024 bytes at a time
 359            chunk = file.read(1024*1024)
 360            h.update(chunk)
 361
 362    # return the hex representation of digest
 363    return h.hexdigest()
 364
 365 def find_dupes_dir(BASE_OLD, BASE_NEW):
 366         dups = {}
 367         fdupes = {}
 368         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 369         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 370         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 371                 #print('Scanning %s...' % rootbase)
 372                 for filename in fileListB:
 373                         path = os.path.join(rootbase, filename)
 374                         if os.path.islink(path):
 375                                 continue
 376                         # Calculate hash
 377                         file_hash = hash_file(path)
 378                         dups[file_hash] = path
 379
 380         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 381                 #print('Scanning %s...' % roottarget)
 382                 for filename in fileListT:
 383                         # Get the path to the file
 384                         path = os.path.join(roottarget, filename)
 385                         if os.path.islink(path):
 386                                 continue
 387                         # Calculate hash
 388                         file_hash = hash_file(path)
 389                         # Add or append the file path
 390                         if file_hash in dups:
 391                                 BaseStr = dups.get(file_hash)
 392                                 Baseloc = path.find('/')
 393                                 TarLoc = BaseStr.find('/')
 394                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 395                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 396                                         fdupes[path] = BaseStr
 397         logging.info('Total Duplicate files %d' % (len(fdupes)))
 398         return fdupes
 399
 400
 401 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 402         dups = {}
 403         fdupes = {}
 404         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 405
 406         for filename in fileListB:
 407                 Src_File = BASE_OLD+'/'+filename
 408                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 409                         continue
 410                 # Calculate hash
 411                 file_hash = hash_file(Src_File)
 412                 dups[file_hash] = Src_File
 413
 414
 415         for filename in fileListT:
 416                 Dest_File = BASE_NEW+'/'+filename
 417                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 418                         continue
 419                 # Calculate hash
 420                 file_hash = hash_file(Dest_File)
 421                 if file_hash in dups:
 422                         BaseStr = dups.get(file_hash)
 423                         Baseloc = BaseStr.find('/')
 424                         if not BaseStr[Baseloc:] == filename:
 425                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 426                                 fdupes[BaseStr] = filename
 427
 428         logging.info('Total Duplicate files %d' % (len(fdupes)))
 429         return fdupes
 430
 431 def SS_UpdateSize(src_file, dst_file):
 432         global MEM_REQ
 433         global MEM_FILE
 434         oldsize_d= os.path.getsize(src_file)
 435         newsize_d= os.path.getsize(dst_file)
 436         if oldsize_d >= newsize_d:
 437                 Max = newsize_d
 438         else:
 439                 Max = oldsize_d
 440         if MEM_REQ < Max:
 441                 MEM_REQ = Max
 442                 MEM_FILE = dst_file
 443
 444
 445
 446 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 447         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 448         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 449
 450         # First let's fill up these categories
 451         files_new = []
 452         files_removed = []
 453         Dir_removed = []
 454         Dir_Added = []
 455         files_changed = []
 456         files_unchanged = []
 457         files_renamed = []
 458         File_Attibutes = []
 459         Sym_Attibutes = []
 460
 461         files_Del_List = {}
 462         files_New_List = {}
 463         MyDict_Patches = {}
 464
 465
 466
 467         PWD = os.getcwd()
 468
 469         # Generate NEW List
 470         for elt in New_files:
 471                 if elt not in Old_files:
 472                         files_new.append(elt)
 473                         logging.info('New files %s' % elt)
 474
 475         # Generate Delete List
 476         for elt in Old_files:
 477                 if elt not in New_files:
 478                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 479                         files_removed.append(elt)
 480                         logging.info('Old files %s' % elt)
 481
 482
 483         for elt in Old_dirs:
 484                 #print('List of Old Dirs %s' % elt)
 485                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 486                 if elt not in New_dirs:
 487                         Dir_removed.append(elt)
 488                         logging.info('Old Dirs %s' % elt+'/')
 489
 490         for elt in New_dirs:
 491                 if elt not in Old_dirs:
 492                         Dir_Added.append(elt)
 493                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 494
 495         # What files have changed contents but not name/path?
 496         for elt in New_files:
 497                 if elt in Old_files:
 498                         #Both are symbolic linkes and they differ
 499                         src_file = BASE_OLD+'/'+elt
 500                         dst_file = BASE_NEW+'/'+elt
 501                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 502                         if os.path.islink(src_file) and os.path.islink(dst_file):
 503                                 if not os.readlink(src_file) == os.readlink(dst_file):
 504                                         files_changed.append(elt)
 505                                         #print('%d Sym link files changed' % len(files_changed))
 506                                         logging.info('Sym links Changed - %s' % elt)
 507                                 else:
 508                                         files_unchanged.append(elt)
 509                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 510                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 511                                 if not filecmp.cmp(src_file, dst_file):
 512                                         files_changed.append(elt)
 513                                         #print('%d Normal files changed' % len(files_changed))
 514                                         #print('Files Changed - %s' % elt)
 515                                 else:
 516                                         files_unchanged.append(elt)
 517                         #File types differ between BASE and TARGET
 518                         else:
 519                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 520                                 #Both file types have changed and they differ
 521                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 522                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 523                                 files_removed.append(elt)
 524                                 files_new.append(elt)
 525
 526
 527         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 528         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 529
 530         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 531         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 532                 with open(VERBATIM_LIST, 'r') as F_News:
 533                         lines = set(F_News.read().splitlines())
 534                 for line in lines:
 535                         if line in files_changed:
 536                                 files_changed.remove(line)
 537                                 files_removed.append(line)
 538                         if line in files_new:
 539                                 files_new.remove(line)
 540
 541         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 542         if SUPPORT_RENAME == "TRUE":
 543                 for elt in files_removed:
 544                         if os.path.isfile(BASE_OLD+'/'+elt):
 545                                 FileName = path_leaf(elt)
 546                                 entries = re.split('[0-9]' , FileName)
 547                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 548                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 549                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 550                                 if len(entries[0]) > 0:
 551                                         files_Del_List.update({entries[0]: elt})
 552
 553                 for elt in files_new:
 554                         if os.path.isfile(BASE_NEW+'/'+elt):
 555                                 FileName = path_leaf(elt)
 556                                 entries = re.split('[0-9]' , FileName)
 557                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 558                                 if len(entries[0]) > 0:
 559                                         files_New_List.update({entries[0]: elt})
 560
 561                 for key, value in files_Del_List.iteritems():
 562                         #print('Key value pair -%s -%s' % (key, value))
 563                         if key in files_New_List:
 564                                 # this file is the same name in both!
 565                                 src_file = BASE_OLD+'/'+value
 566                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 567                                 olddirpath = path_head(files_New_List[key])
 568                                 newdirpath = path_head(value)
 569                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 570                                         logging.debug('Cannot diff as one of them is Symlink')
 571                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 572                                         logging.debug('Cannot diff as one of them is dir')
 573                                 else:
 574                                         #Pick the best diff of same type and diff names
 575                                         files_renamed.append([files_New_List[key], value])
 576                                         files_removed.remove(value)
 577                                         files_new.remove(files_New_List[key])
 578
 579         '''
 580         Patch Section
 581                 Partition.txt contains Protocol for UPI
 582                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 583         '''
 584         Sym_Diff_Cnt = 0
 585         Sym_New_Cnt = 0;
 586         Del_Cnt = 0
 587         New_Cnt = 0
 588         Diff_Cnt = 0
 589         Move_Cnt = 0
 590         Verbatim_Cnt = 0
 591         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 592         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 593         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 594
 595         print("writing diff'ed changed files...")
 596         for elt in files_changed:
 597                 dst_file = BASE_NEW+'/'+elt
 598                 src_file = BASE_OLD+'/'+elt
 599                 #Both files are symbolic links and they differ
 600                 if os.path.islink(dst_file) and os.path.islink(src_file):
 601                 #Both are symlinks and they differ
 602                         logging.debug(' File Changed is Link %s ' % dst_file)
 603                         patch = os.readlink(dst_file)
 604                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 605                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 606                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 607                 #Both are NORMAL files and they differ
 608                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 609                         #Both are files and they differ
 610                         Diff_Cnt = Diff_Cnt + 1
 611                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 612                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 613                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 614                         SS_UpdateSize(src_file, dst_file)
 615
 616                         FORMAT = "REG"
 617                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 618                         if ret is not 0:
 619                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 620                                 files_new.append(elt)
 621                                 Diff_Cnt = Diff_Cnt - 1
 622                         else:
 623                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 624
 625                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 626                 #Both differ but they are of diff types
 627                 else:
 628                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 629                         files_removed.append(elt)
 630                         files_new.append(elt)
 631
 632         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 633         for oldpath, newpath in fdupes.iteritems():
 634                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 635
 636         for elt in files_removed:
 637                 src_file = BASE_OLD+'/'+elt
 638                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 639                 if src_file in fdupes.keys():
 640                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 641                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 642                         Move_Cnt = Move_Cnt + 1
 643                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 644                         files_removed.remove(elt)
 645                         files_new.remove(fdupes[src_file])
 646
 647         #Should be placed after removing duplicates, else they will be filtered here.
 648         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 649         #       First Step: Sort & Filter out unwanted files
 650         # Minimum condition used is,
 651         #       1. File name should match 70%
 652         #       2. Extensions should be same
 653         #       3. File name length shd b greater than 3 char
 654         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 655         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 656
 657         Base_DelList = files_removed[:]
 658         Base_NewList = files_new[:]
 659         DelList = sorted(Base_DelList, key=path_leaf)
 660         NewList = sorted(Base_NewList, key=path_leaf)
 661         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 662
 663         Filter1 = []
 664         Filter2 = []
 665         #Remove unwanted items which we cant make diff with for rename logic
 666         for file in DelList:
 667                 if os.path.islink(BASE_OLD+'/'+file):
 668                         continue
 669                 elif os.path.isdir(BASE_OLD+'/'+file):
 670                         continue
 671                 else:
 672                         Filter1.append(file)
 673                         #logging.debug('Sorted del list - %s' % (file))
 674
 675         DelList = Filter1
 676
 677         for file in NewList:
 678                 if os.path.islink(BASE_NEW+'/'+file):
 679                         continue
 680                 elif os.path.isdir(BASE_NEW+'/'+file):
 681                         continue
 682                 elif len(path_leaf(file)) <= 3:
 683                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 684                         continue
 685                 else:
 686                         Filter2.append(file)
 687
 688         NewList = Filter2
 689
 690         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 691
 692         for new_file in NewList:
 693                 R_Flag = 'FALSE';
 694                 DirPathNew = path_head(new_file)
 695                 FileNameNew = path_leaf(new_file)
 696                 DiffSize = 0
 697                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 698                 New_fs = winning_patch_sz
 699                 winning_file = ''
 700
 701                 for del_file in DelList:
 702                         FileNameOld = path_leaf(del_file)
 703                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 704                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 705                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 706                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 707                                 v1 = abs(New_fs-Del_fs)
 708                                 v2 = (New_fs+Del_fs)/2
 709                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 710                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 711                                         continue
 712                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 713                                 R_Flag = 'TRUE';
 714                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 715                                 if (DiffSize < 0.8 * winning_patch_sz):
 716                                         winning_patch_sz = DiffSize
 717                                         winning_file = del_file
 718                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 719                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 720                                 break;
 721                 if len(winning_file) > 0:
 722                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 723                         files_renamed.append([new_file, winning_file])
 724                         DelList.remove(winning_file)
 725                         files_removed.remove(winning_file)
 726                         files_new.remove(new_file)
 727
 728         #********************** Files should NOT be deleted for any such renames ***********************
 729
 730         if SUPPORT_RENAME == "TRUE":
 731                 for elt in files_renamed:
 732                         src_file = BASE_OLD+'/'+elt[1]
 733                         dst_file = BASE_NEW+'/'+elt[0]
 734                         Diff_Cnt = Diff_Cnt + 1
 735                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 736                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 737                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 738                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 739                         # Should be careful of renaming files??
 740                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 741                         # What if folder path has numerics??
 742
 743                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 744                                 #This case never occurs??
 745                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 746                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 747                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 748                                 if filecmp.cmp(src_file, dst_file):
 749                                         Move_Cnt = Move_Cnt + 1
 750                                         Diff_Cnt = Diff_Cnt - 1
 751                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 752                                 else:
 753                                         FORMAT = "REG"
 754                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 755                                         if ret is not 0:
 756                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 757                                                 files_new.append(elt)
 758                                                 Diff_Cnt = Diff_Cnt - 1
 759                                         else:
 760                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 761
 762                                 SS_UpdateSize(src_file, dst_file)
 763                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 764
 765
 766         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 767         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 768
 769         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 770                 with open(VERBATIM_LIST, 'r') as F_News:
 771                         lines = set(F_News.read().splitlines())
 772                 for line in lines:
 773                         if not line in files_new:
 774                                 if os.path.exists(BASE_NEW+'/'+line):
 775                                         files_new.append(line)
 776                                         Verbatim_Cnt = Verbatim_Cnt+1
 777                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 778
 779
 780
 781         for elt in files_removed:
 782                 #if files are part of patches after renaming, we shd remove them as part of removed.
 783                 src_file = BASE_OLD+'/'+elt
 784                 if os.path.islink(src_file):
 785                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 786                 elif os.path.isdir(src_file):
 787                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 788                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 789                 else:
 790                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 791                 logging.debug(' File Deleted %s' % src_file)
 792                 Del_Cnt = Del_Cnt + 1
 793
 794         Dir_removed.sort(reverse=True)
 795         for elt in Dir_removed:
 796                 #if Dir is empty, add it to the removed list.
 797                 src_file = BASE_OLD+'/'+elt
 798                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 799                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 800                 if os.path.isdir(src_file):
 801                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 802                         Del_Cnt = Del_Cnt + 1
 803                         logging.debug(' Dir Deleted- %s' % src_file)
 804
 805
 806         for elt in files_new:
 807                 dst_file = BASE_NEW+'/'+elt
 808                 newfiles_dest_path = 'run/upgrade-sysroot/'
 809                 ensure_dir_exists(newfiles_dest_path)
 810                 if os.path.islink(dst_file):
 811                         patch = os.readlink(dst_file)
 812                         logging.debug(' File New Links %s' % elt)
 813                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 814                         #What if this is only a new sym link and folder already exists??? Should recheck
 815                         destpath = newfiles_dest_path + elt
 816                         if not os.path.exists(path_head(destpath)):
 817                                 os.makedirs(path_head(destpath))
 818                                 logging.info('New SymLink - Adding missing Dir')
 819                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 820                         Sym_New_Cnt = Sym_New_Cnt + 1
 821                 elif os.path.isdir(dst_file): # We create just empty directory here
 822                         destpath = newfiles_dest_path + elt
 823                         if not os.path.exists(destpath):
 824                                 os.makedirs(destpath)
 825                                 logging.debug(' File New Dir %s' % destpath)
 826                                 New_Cnt = New_Cnt + 1
 827                 else:
 828                         New_Cnt = New_Cnt + 1
 829                         destpath = newfiles_dest_path + elt
 830                         destdir = os.path.dirname(destpath)
 831                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 832
 833                         if not os.path.isdir(destdir):
 834                                 try:
 835                                         os.makedirs(destdir)
 836                                 except:
 837                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 838                                         raise
 839
 840                         try:
 841                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 842                                         shutil.copy2(dst_file, destpath)
 843                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 844                         except:
 845                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 846                                 raise
 847
 848         for elt in Dir_Added:
 849                 newfiles_dest_path = 'run/upgrade-sysroot/'
 850                 ensure_dir_exists(newfiles_dest_path)
 851                 destpath = newfiles_dest_path + elt
 852                 if not os.path.exists(destpath):
 853                         os.makedirs(destpath)
 854                         logging.debug(' DirList New Dir %s' % destpath)
 855                         New_Cnt = New_Cnt + 1
 856
 857         #Base directory should be system
 858         print 'Compressing New files'
 859         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 860                 WorkingDir = os.getcwd()
 861                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 862                 logging.info('Curr Working Dir - %s' % os.getcwd())
 863                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 864                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 865                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 866                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 867                 os.chdir(WorkingDir)
 868                 shutil.rmtree(NEW_FILES_PATH)
 869                 # use 7z a system.7z ./*
 870
 871         #logging.info('%d Dir to be removed' % len(Dir_removed))
 872         logging.info('%d files unchanged' % len(files_unchanged))
 873         logging.info('%d files files_renamed' % len(files_renamed))
 874         logging.info('%d files NEW' % len(files_new))
 875         logging.info('%d File attr' % len(File_Attibutes))
 876         logging.info('%d Sym attr' % len(Sym_Attibutes))
 877         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 878         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 879
 880         #There could be duplicates, TODO, can check before adding..
 881         ATTR_FILE_D = open(ATTR_FILE,'a+')
 882         for elt in File_Attibutes:
 883                 ATTR_FILE_D.write(elt)
 884         for elt in Sym_Attibutes:
 885                 ATTR_FILE_D.write(elt)
 886
 887         ATTR_FILE_D.close()
 888
 889         Partition_Doc_SymLinks.close()
 890         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 891         Partition_Doc.write(Partition_Read_SymLinks.read())
 892         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 893         Partition_Doc_SymLinks.close()
 894         Partition_Doc.close()
 895         os.remove(SymLinkDoc)
 896
 897         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 898                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 899                 logging.info('No Delta Generated for %s' % PART_NAME)
 900                 shutil.rmtree(OUT_DIR)
 901
 902
 903 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 904
 905         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 906         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 907         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 908         temp_apk = '../'+g_output_dir+'/'+b_apk
 909         Patch = 'Patch_'+b_apk
 910         ensure_dir_exists(Patch)
 911         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 912
 913         #Size issue on Device side?? shd check this
 914         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 915         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 916                 lines = set(f_new.read().splitlines())
 917                 for line in lines:
 918                         #print('Action ==> %s' % line)
 919                         #Action, Path, Patch = line.split('|')
 920                         Items = line.split('|')
 921                         Action = Items[0]
 922                         Path = Items[1]
 923                         ActualPath = a_folder+'/'+Path
 924                         PatchPath = Patch+'/'+Path
 925                         SrcPath = g_output_dir+'/'+path_leaf(Path)
 926                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 927                         if line[0] == 'c':
 928                                 patchName = g_output_dir+'/'+Items[2]
 929                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 930                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
 931                                 WorkingDir = os.getcwd()
 932                                 os.chdir(WorkingDir+"/"+"temp_a")
 933                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 934                                 os.chdir(WorkingDir)
 935                         elif line[0] == 's':
 936                                 WorkingDir = os.getcwd()
 937                                 os.chdir(WorkingDir+"/"+"temp_a")
 938                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 939                                 os.chdir(WorkingDir)
 940                         else:
 941                                 print('Apply_Container_Delta - Unknown Error')
 942         #print('Touch all files and set common attributes for DIFF generation')
 943         WorkingDir = os.getcwd()
 944         os.chdir(WorkingDir+"/"+Patch)
 945
 946         CONTAINER_DATE = '200011111111.11'
 947         CONTAINER_MODE = '0755'
 948         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
 949         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
 950         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
 951
 952         print 'Update Intermediate Archive'
 953         #subprocess.call(['zip','-ryX', b_apk, '*'])
 954         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
 955         os.chdir(WorkingDir)
 956         #print('Apply Path completed - Now create diff for this and place in patch folder')
 957         #print os.getcwd()
 958         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
 959         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
 960         patchLoc = '%s/%s' % (g_output_dir, patchName)
 961
 962         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
 963
 964         #Only on HOST... for testing
 965         if TEST_MODE == 'TRUE':
 966                 UpgradedName = '%s_Upgraded' % (b_apk)
 967                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
 968
 969         #This is file only with NEWS and empty diffs and same files.
 970         if TEST_MODE == 'FALSE':
 971                 os.remove(g_output_dir+'/'+b_apk)
 972                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
 973                 shutil.rmtree(Patch)
 974
 975 def IsSymlink(info):
 976   return (info.external_attr >> 16) == 0120777
 977
 978 def NewFiles(src, dest):
 979         print src,dest
 980         subprocess.call(['cp','-rp', src,dest])
 981     #try:
 982                 #shutil.copytree(src, dest)
 983     #except OSError as e:
 984         # If the error was caused because the source wasn't a directory
 985         #if e.errno == errno.ENOTDIR:
 986             #shutil.copy2(src, dest)
 987         #else:
 988             #print('Directory not copied. Error: %s' % e)
 989
 990 def measure_two_filediffs(src, dst):
 991         patchLoc = 'temp.patch'
 992         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
 993         result_size = os.path.getsize(patchLoc)
 994         os.remove(patchLoc)
 995         return result_size
 996
 997 def Get_Files(path):
 998         all_files = []
 999         all_dirs = []
1000
1001         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1002                 for directory in directories:
1003                         #DirName = os.path.join(root+'/',directory)
1004                         DirName = os.path.join(root,directory)
1005                         if os.path.islink(DirName):
1006                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1007                                 all_files.append(os.path.relpath(DirName, path))
1008                         elif not os.listdir(DirName):
1009                                 #print('*****Empty Directory******* -%s', DirName)
1010                                 #This should NOT be appended ??? Empty dir shd b considered
1011                                 all_dirs.append(os.path.relpath(DirName, path))
1012                         else:
1013                                 all_dirs.append(os.path.relpath(DirName, path))
1014                 for filename in filenames:
1015                         FileName = os.path.join(root,filename)
1016                         all_files.append(os.path.relpath(FileName, path))
1017
1018         all_files.sort()
1019         all_dirs.sort()
1020         return all_files, all_dirs
1021
1022
1023 USAGE_DOCSTRING = """
1024       Generate Delta using BASEOLD AND BASE NEW
1025           Attributes is optional
1026           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1027 """
1028
1029 def Usage(docstring):
1030   print docstring.rstrip("\n")
1031   print COMMON_DOCSTRING
1032
1033
1034
1035 if __name__ == '__main__':
1036         main()
1037