mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49 def global_paths():
  50         global DIFF_UTIL
  51         global ZIPUTIL
  52         global NEW_FILES_PATH
  53         global NEW_FILES_ZIP_NAME
  54         global SYMLINK_TYPE
  55         global ATTR_DOC_EXT
  56         global SYMLINK_DOC_NAME
  57         global DIFF_PREFIX
  58         global DIFF_SUFFIX
  59         global SUPPORT_RENAME
  60         global NEW_PREFIX
  61         global DIFFPATCH_UTIL
  62         global SUPPORT_CONTAINERS
  63         global FULL_IMG
  64         global DELTA_IMG
  65         global DELTA_IMG_AB
  66         global DELTA_FS
  67         global EXTRA
  68         global COMMON_BIN_PATH
  69         global MEM_REQ
  70         global EMPTY
  71         global VERBATIM_LIST
  72         global MEM_FILE
  73
  74 COMMON_BIN_PATH = "../../common/bin/"
  75 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  76 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  77 #ZIPUTIL = "p7zip "
  78 ZIPUTIL = "7z -mf=off a system.7z "
  79 NEW_FILES_PATH = "run/upgrade-sysroot"
  80 NEW_FILES_ZIP_NAME = "system.7z"
  81 SYMLINK_TYPE = "SYM"
  82 ATTR_DOC_EXT = "_attr.txt"
  83 SYMLINK_DOC_NAME = "_sym.txt"
  84 PART_DOC_EXT = ".txt"
  85 DIFF_PREFIX = "diff"
  86 DIFF_SUFFIX = ".delta"
  87 NEW_PREFIX = 'new'
  88 FULL_IMG = "FULL_IMG"
  89 DELTA_IMG = "DELTA_IMG"
  90 DELTA_IMG_AB = "DELTA_IMG_AB"
  91 DELTA_FS = "DELTA_FS"
  92 EXTRA = "EXTRA"
  93 PRE_UA = "PRE_UA"
  94 LOGFILE = "Delta.log"
  95 VERBATIM_LIST = "Verbatim_List.txt"
  96 EMPTY = ""
  97 MEM_REQ = 0
  98 MEM_FILE = "NULL"
  99 COMPRESSION_LZMA = "lzma"
 100 COMPRESSION_BROTLI = "brotli"
 101
 102 SUPPORT_RENAME = "TRUE" #Use appropriate name
 103 SUPPORT_CONTAINERS = "FALSE"
 104 SUPPORT_VERBATIM = "TRUE"
 105
 106 TEST_MODE = "FALSE"
 107
 108 def main():
 109         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 110         global AttributeFile
 111         global GenerateDiffAttr
 112         try:
 113
 114                 if len(sys.argv) < 5:
 115                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 116                 UPDATE_TYPE = sys.argv[1]
 117                 PART_NAME = sys.argv[2]  # lets make this also optional
 118
 119                 BASE_OLD = sys.argv[3]
 120                 BASE_NEW = sys.argv[4]
 121                 OUT_DIR = sys.argv[5]
 122                 ATTR_OLD = EMPTY
 123                 ATTR_NEW = EMPTY
 124                 UPDATE_CFG_PATH = EMPTY
 125                 GenerateDiffAttr = "FALSE"
 126                 if UPDATE_TYPE == DELTA_FS:
 127                         #instead of arguments check it in outdirectory ?
 128                         if len(sys.argv) == 9:
 129                                 ATTR_OLD = sys.argv[6]
 130                                 ATTR_NEW = sys.argv[7]
 131                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 132                                 GenerateDiffAttr = "TRUE"
 133
 134                 elif UPDATE_TYPE in [DELTA_IMG, DELTA_IMG_AB, FULL_IMG]:
 135                         if len(sys.argv) == 7:
 136                                 #Use path in better way
 137                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 138
 139                 global DIFF_UTIL
 140                 global DIFFPATCH_UTIL
 141                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 142                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 143                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 144                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 145                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 146                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 147                                 sys.exit(1)
 148
 149                 start = datetime.datetime.now().time()
 150                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 151                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 152
 153                 ensure_dir_exists(OUT_DIR)
 154                 if GenerateDiffAttr == "TRUE":
 155                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 156                                 print >> sys.stderr, "Attributes missing -- ABORT"
 157                                 sys.exit(1)
 158
 159
 160                 # Should check if APT is supported on other linux flavours
 161                 cache = apt.Cache()
 162                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 163                         logging.info ('Basic utils installed')
 164                 else:
 165                         print >> sys.stderr, "Basic utils missing -- ABORT"
 166                         sys.exit(1)
 167
 168                 if UPDATE_TYPE == FULL_IMG:
 169                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 170                 elif UPDATE_TYPE == DELTA_IMG:
 171                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
 172                 elif UPDATE_TYPE == DELTA_IMG_AB:
 173                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
 174                 elif UPDATE_TYPE == DELTA_FS:
 175                         AttributeFile = ATTR_NEW
 176                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 177                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 178                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 179                         New_files, New_dirs = Get_Files(BASE_NEW)
 180                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 181
 182                         if not UPDATE_CFG_PATH == EMPTY:
 183                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 184
 185
 186                 elif UPDATE_TYPE == EXTRA:
 187                         print('UPDATE_TYPE ---- EXTRA')
 188                 elif UPDATE_TYPE == PRE_UA:
 189                         print('UPDATE_TYPE ---- PRE_UA')
 190                 else:
 191                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 192
 193                 if GenerateDiffAttr == "TRUE":
 194                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 195                                 os.remove(ATTR_OLD)
 196                                 os.remove(ATTR_NEW)
 197                 end = datetime.datetime.now().time()
 198
 199                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 200                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 201                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 202                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 203
 204         except:
 205                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 206                 raise
 207
 208
 209 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 210         f = open(UPDATE_CFG_PATH, 'r')
 211         lines = f.readlines()
 212         f.close()
 213         f = open(UPDATE_CFG_PATH, 'w')
 214         for line in lines:
 215                 ConfigItems = line.split()
 216                 if ConfigItems[0] == DELTA_BIN:
 217                         DELTA = ConfigItems[1]
 218                         logging.info ('Updating %s config' % DELTA_BIN)
 219                         line = line.rstrip('\n')
 220                         Value = MEM_REQ
 221                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 222                         f.write(line)
 223                 else:
 224                         f.write(line)
 225         f.close()
 226
 227 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
 228         #for sizes
 229
 230         oldsize_d= os.path.getsize(BASE_OLD)
 231         newsize_d= os.path.getsize(BASE_NEW)
 232         SHA_BIN_DEST= hash_file(BASE_NEW)
 233         SHA_BIN_BASE=hash_file(BASE_OLD)
 234
 235         #incase UPDATE CFG is empty
 236         DELTA = DELTA_BIN
 237         SS_UpdateSize(BASE_OLD, BASE_NEW)
 238         #Should throw error if PART NAME NOT found??
 239         if not UPDATE_CFG_PATH == EMPTY:
 240                 f = open(UPDATE_CFG_PATH, 'r')
 241                 lines = f.readlines()
 242                 f.close()
 243                 f = open(UPDATE_CFG_PATH, 'w')
 244                 for line in lines:
 245                         ConfigItems = line.split()
 246                         if ConfigItems[0] == DELTA_BIN:
 247                                 logging.info ('Updating %s config' % DELTA_BIN)
 248                                 DELTA = ConfigItems[1]
 249                                 line = line.rstrip('\n')
 250                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 251                                 f.write(line)
 252                         else:
 253                                 f.write(line)
 254                 f.close()
 255
 256         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 257         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 258         subprocess.call([DIFF_UTIL,"-c",COMPRESSION_METHOD,BASE_OLD,BASE_NEW,patchLoc])
 259
 260
 261
 262 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 263         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 264         oldsize_d= os.path.getsize(BASE_OLD)
 265         newsize_d= os.path.getsize(BASE_NEW)
 266         SHA_BIN_DEST= hash_file(BASE_NEW)
 267         SHA_BIN_BASE=hash_file(BASE_OLD)
 268         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 269         SS_UpdateSize(BASE_OLD, BASE_NEW)
 270
 271         if not UPDATE_CFG_PATH == EMPTY:
 272                 f = open(UPDATE_CFG_PATH, 'r')
 273                 lines = f.readlines()
 274                 f.close()
 275                 f = open(UPDATE_CFG_PATH, 'w')
 276                 for line in lines:
 277                         ConfigItems = line.split()
 278                         if ConfigItems[0] == DELTA_BIN:
 279                                 logging.info ('Updating %s config' % DELTA_BIN)
 280                                 DELTA = ConfigItems[1]
 281                                 line = line.rstrip('\n')
 282                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 283                                 f.write(line)
 284                         else:
 285                                 f.write(line)
 286                 f.close()
 287
 288 def zipdir(path, zip):
 289     for root, dirs, files in os.walk(path):
 290         for file in files:
 291             zip.write(os.path.join(root, file))
 292
 293 def ensure_dir_exists(path):
 294         if not os.path.exists(path):
 295                 os.makedirs(path)
 296                 #shutil.rmtree(path)
 297         #os.makedirs(path)
 298
 299
 300 def path_leaf(path):
 301     head, tail = ntpath.split(path) #This is for windows?? Recheck
 302     return tail
 303
 304 def path_head(path):
 305     head, tail = ntpath.split(path)
 306     return head
 307
 308 def difflines(list1, list2):
 309     c = set(list1).union(set(list2))
 310     d = set(list1).intersection(set(list2))
 311     return list(c-d)
 312
 313 #Creating Diff between OLD and NEW attribute files v12
 314 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 315         if GenerateDiffAttr == "FALSE":
 316                 return
 317         with open(ATTR_OLD, 'r') as f_old:
 318                 lines1 = set(f_old.read().splitlines())
 319
 320         with open(ATTR_NEW, 'r') as f_new:
 321                 lines2 = set(f_new.read().splitlines())
 322
 323         lines = difflines(lines2, lines1)
 324         with open(ATTR_FILE, 'w+') as file_out:
 325                 for line in lines:
 326                         if line not in lines1:
 327                                 logging.info('Diff_AttrFiles - %s' % line)
 328                                 file_out.write(line+'\n')
 329
 330         f_new.close()
 331         f_old.close()
 332         file_out.close()
 333
 334
 335
 336 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 337         #Full File Path should MATCH
 338         if GenerateDiffAttr == "FALSE":
 339                 return
 340         FilePath = '"/'+RequestedPath+'"'
 341         #print ('FilePath - %s'% (FilePath))
 342         with open(AttributeFile) as f:
 343                 for line in f:
 344                         if FilePath in line:
 345                                 if Type == SYMLINK_TYPE:
 346                                         Sym_Attibutes.append(line)
 347                                 else:
 348                                         File_Attibutes.append(line)
 349
 350
 351 '''This function returns the SHA-1 hash of the file passed into it'''
 352 def hash_file(filename):
 353
 354    # make a hash object
 355    h = hashlib.sha1()
 356
 357    # open file for reading in binary mode
 358    with open(filename,'rb') as file:
 359        # loop till the end of the file
 360        chunk = 0
 361        while chunk != b'':
 362            # read only 1024 bytes at a time
 363            chunk = file.read(1024*1024)
 364            h.update(chunk)
 365
 366    # return the hex representation of digest
 367    return h.hexdigest()
 368
 369 def find_dupes_dir(BASE_OLD, BASE_NEW):
 370         dups = {}
 371         fdupes = {}
 372         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 373         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 374         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 375                 #print('Scanning %s...' % rootbase)
 376                 for filename in fileListB:
 377                         path = os.path.join(rootbase, filename)
 378                         if os.path.islink(path):
 379                                 continue
 380                         # Calculate hash
 381                         file_hash = hash_file(path)
 382                         dups[file_hash] = path
 383
 384         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 385                 #print('Scanning %s...' % roottarget)
 386                 for filename in fileListT:
 387                         # Get the path to the file
 388                         path = os.path.join(roottarget, filename)
 389                         if os.path.islink(path):
 390                                 continue
 391                         # Calculate hash
 392                         file_hash = hash_file(path)
 393                         # Add or append the file path
 394                         if file_hash in dups:
 395                                 BaseStr = dups.get(file_hash)
 396                                 Baseloc = path.find('/')
 397                                 TarLoc = BaseStr.find('/')
 398                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 399                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 400                                         fdupes[path] = BaseStr
 401         logging.info('Total Duplicate files %d' % (len(fdupes)))
 402         return fdupes
 403
 404
 405 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 406         dups = {}
 407         fdupes = {}
 408         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 409
 410         for filename in fileListB:
 411                 Src_File = BASE_OLD+'/'+filename
 412                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 413                         continue
 414                 # Calculate hash
 415                 file_hash = hash_file(Src_File)
 416                 dups[file_hash] = Src_File
 417
 418
 419         for filename in fileListT:
 420                 Dest_File = BASE_NEW+'/'+filename
 421                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 422                         continue
 423                 # Calculate hash
 424                 file_hash = hash_file(Dest_File)
 425                 if file_hash in dups:
 426                         BaseStr = dups.get(file_hash)
 427                         Baseloc = BaseStr.find('/')
 428                         if not BaseStr[Baseloc:] == filename:
 429                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 430                                 fdupes[BaseStr] = filename
 431
 432         logging.info('Total Duplicate files %d' % (len(fdupes)))
 433         return fdupes
 434
 435 def SS_UpdateSize(src_file, dst_file):
 436         global MEM_REQ
 437         global MEM_FILE
 438         oldsize_d= os.path.getsize(src_file)
 439         newsize_d= os.path.getsize(dst_file)
 440         if oldsize_d >= newsize_d:
 441                 Max = newsize_d
 442         else:
 443                 Max = oldsize_d
 444         if MEM_REQ < Max:
 445                 MEM_REQ = Max
 446                 MEM_FILE = dst_file
 447
 448
 449
 450 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 451         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 452         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 453
 454         # First let's fill up these categories
 455         files_new = []
 456         files_removed = []
 457         Dir_removed = []
 458         Dir_Added = []
 459         files_changed = []
 460         files_unchanged = []
 461         files_renamed = []
 462         File_Attibutes = []
 463         Sym_Attibutes = []
 464
 465         files_Del_List = {}
 466         files_New_List = {}
 467         MyDict_Patches = {}
 468
 469
 470
 471         PWD = os.getcwd()
 472
 473         # Generate NEW List
 474         for elt in New_files:
 475                 if elt not in Old_files:
 476                         files_new.append(elt)
 477                         logging.info('New files %s' % elt)
 478
 479         # Generate Delete List
 480         for elt in Old_files:
 481                 if elt not in New_files:
 482                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 483                         files_removed.append(elt)
 484                         logging.info('Old files %s' % elt)
 485
 486
 487         for elt in Old_dirs:
 488                 #print('List of Old Dirs %s' % elt)
 489                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 490                 if elt not in New_dirs:
 491                         Dir_removed.append(elt)
 492                         logging.info('Old Dirs %s' % elt+'/')
 493
 494         for elt in New_dirs:
 495                 if elt not in Old_dirs:
 496                         Dir_Added.append(elt)
 497                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 498
 499         # What files have changed contents but not name/path?
 500         for elt in New_files:
 501                 if elt in Old_files:
 502                         #Both are symbolic linkes and they differ
 503                         src_file = BASE_OLD+'/'+elt
 504                         dst_file = BASE_NEW+'/'+elt
 505                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 506                         if os.path.islink(src_file) and os.path.islink(dst_file):
 507                                 if not os.readlink(src_file) == os.readlink(dst_file):
 508                                         files_changed.append(elt)
 509                                         #print('%d Sym link files changed' % len(files_changed))
 510                                         logging.info('Sym links Changed - %s' % elt)
 511                                 else:
 512                                         files_unchanged.append(elt)
 513                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 514                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 515                                 if not filecmp.cmp(src_file, dst_file):
 516                                         files_changed.append(elt)
 517                                         #print('%d Normal files changed' % len(files_changed))
 518                                         #print('Files Changed - %s' % elt)
 519                                 else:
 520                                         files_unchanged.append(elt)
 521                         #File types differ between BASE and TARGET
 522                         else:
 523                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 524                                 #Both file types have changed and they differ
 525                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 526                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 527                                 files_removed.append(elt)
 528                                 files_new.append(elt)
 529
 530
 531         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 532         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 533
 534         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 535         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 536                 with open(VERBATIM_LIST, 'r') as F_News:
 537                         lines = set(F_News.read().splitlines())
 538                 for line in lines:
 539                         if line in files_changed:
 540                                 files_changed.remove(line)
 541                                 files_removed.append(line)
 542                         if line in files_new:
 543                                 files_new.remove(line)
 544
 545         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 546         if SUPPORT_RENAME == "TRUE":
 547                 for elt in files_removed:
 548                         if os.path.isfile(BASE_OLD+'/'+elt):
 549                                 FileName = path_leaf(elt)
 550                                 entries = re.split('[0-9]' , FileName)
 551                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 552                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 553                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 554                                 if len(entries[0]) > 0:
 555                                         files_Del_List.update({entries[0]: elt})
 556
 557                 for elt in files_new:
 558                         if os.path.isfile(BASE_NEW+'/'+elt):
 559                                 FileName = path_leaf(elt)
 560                                 entries = re.split('[0-9]' , FileName)
 561                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 562                                 if len(entries[0]) > 0:
 563                                         files_New_List.update({entries[0]: elt})
 564
 565                 for key, value in files_Del_List.iteritems():
 566                         #print('Key value pair -%s -%s' % (key, value))
 567                         if key in files_New_List:
 568                                 # this file is the same name in both!
 569                                 src_file = BASE_OLD+'/'+value
 570                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 571                                 olddirpath = path_head(files_New_List[key])
 572                                 newdirpath = path_head(value)
 573                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 574                                         logging.debug('Cannot diff as one of them is Symlink')
 575                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 576                                         logging.debug('Cannot diff as one of them is dir')
 577                                 else:
 578                                         #Pick the best diff of same type and diff names
 579                                         files_renamed.append([files_New_List[key], value])
 580                                         files_removed.remove(value)
 581                                         files_new.remove(files_New_List[key])
 582
 583         '''
 584         Patch Section
 585                 Partition.txt contains Protocol for UPI
 586                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 587         '''
 588         Sym_Diff_Cnt = 0
 589         Sym_New_Cnt = 0;
 590         Del_Cnt = 0
 591         New_Cnt = 0
 592         Diff_Cnt = 0
 593         Move_Cnt = 0
 594         Verbatim_Cnt = 0
 595         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 596         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 597         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 598
 599         print("writing diff'ed changed files...")
 600         for elt in files_changed:
 601                 dst_file = BASE_NEW+'/'+elt
 602                 src_file = BASE_OLD+'/'+elt
 603                 #Both files are symbolic links and they differ
 604                 if os.path.islink(dst_file) and os.path.islink(src_file):
 605                 #Both are symlinks and they differ
 606                         logging.debug(' File Changed is Link %s ' % dst_file)
 607                         patch = os.readlink(dst_file)
 608                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 609                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 610                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 611                 #Both are NORMAL files and they differ
 612                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 613                         #Both are files and they differ
 614                         Diff_Cnt = Diff_Cnt + 1
 615                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 616                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 617                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 618                         SS_UpdateSize(src_file, dst_file)
 619
 620                         FORMAT = "REG"
 621                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 622                         if ret is not 0:
 623                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 624                                 files_new.append(elt)
 625                                 Diff_Cnt = Diff_Cnt - 1
 626                         else:
 627                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 628
 629                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 630                 #Both differ but they are of diff types
 631                 else:
 632                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 633                         files_removed.append(elt)
 634                         files_new.append(elt)
 635
 636         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 637         for oldpath, newpath in fdupes.iteritems():
 638                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 639
 640         for elt in files_removed:
 641                 src_file = BASE_OLD+'/'+elt
 642                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 643                 if src_file in fdupes.keys():
 644                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 645                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 646                         Move_Cnt = Move_Cnt + 1
 647                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 648                         files_removed.remove(elt)
 649                         files_new.remove(fdupes[src_file])
 650
 651         #Should be placed after removing duplicates, else they will be filtered here.
 652         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 653         #       First Step: Sort & Filter out unwanted files
 654         # Minimum condition used is,
 655         #       1. File name should match 70%
 656         #       2. Extensions should be same
 657         #       3. File name length shd b greater than 3 char
 658         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 659         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 660
 661         Base_DelList = files_removed[:]
 662         Base_NewList = files_new[:]
 663         DelList = sorted(Base_DelList, key=path_leaf)
 664         NewList = sorted(Base_NewList, key=path_leaf)
 665         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 666
 667         Filter1 = []
 668         Filter2 = []
 669         #Remove unwanted items which we cant make diff with for rename logic
 670         for file in DelList:
 671                 if os.path.islink(BASE_OLD+'/'+file):
 672                         continue
 673                 elif os.path.isdir(BASE_OLD+'/'+file):
 674                         continue
 675                 else:
 676                         Filter1.append(file)
 677                         #logging.debug('Sorted del list - %s' % (file))
 678
 679         DelList = Filter1
 680
 681         for file in NewList:
 682                 if os.path.islink(BASE_NEW+'/'+file):
 683                         continue
 684                 elif os.path.isdir(BASE_NEW+'/'+file):
 685                         continue
 686                 elif len(path_leaf(file)) <= 3:
 687                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 688                         continue
 689                 else:
 690                         Filter2.append(file)
 691
 692         NewList = Filter2
 693
 694         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 695
 696         for new_file in NewList:
 697                 R_Flag = 'FALSE';
 698                 DirPathNew = path_head(new_file)
 699                 FileNameNew = path_leaf(new_file)
 700                 DiffSize = 0
 701                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 702                 New_fs = winning_patch_sz
 703                 winning_file = ''
 704
 705                 for del_file in DelList:
 706                         FileNameOld = path_leaf(del_file)
 707                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 708                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 709                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 710                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 711                                 v1 = abs(New_fs-Del_fs)
 712                                 v2 = (New_fs+Del_fs)/2
 713                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 714                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 715                                         continue
 716                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 717                                 R_Flag = 'TRUE';
 718                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 719                                 if (DiffSize < 0.8 * winning_patch_sz):
 720                                         winning_patch_sz = DiffSize
 721                                         winning_file = del_file
 722                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 723                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 724                                 break;
 725                 if len(winning_file) > 0:
 726                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 727                         files_renamed.append([new_file, winning_file])
 728                         DelList.remove(winning_file)
 729                         files_removed.remove(winning_file)
 730                         files_new.remove(new_file)
 731
 732         #********************** Files should NOT be deleted for any such renames ***********************
 733
 734         if SUPPORT_RENAME == "TRUE":
 735                 for elt in files_renamed:
 736                         src_file = BASE_OLD+'/'+elt[1]
 737                         dst_file = BASE_NEW+'/'+elt[0]
 738                         Diff_Cnt = Diff_Cnt + 1
 739                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 740                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 741                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 742                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 743                         # Should be careful of renaming files??
 744                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 745                         # What if folder path has numerics??
 746
 747                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 748                                 #This case never occurs??
 749                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 750                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 751                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 752                                 if filecmp.cmp(src_file, dst_file):
 753                                         Move_Cnt = Move_Cnt + 1
 754                                         Diff_Cnt = Diff_Cnt - 1
 755                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 756                                 else:
 757                                         FORMAT = "REG"
 758                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 759                                         if ret is not 0:
 760                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 761                                                 files_new.append(elt)
 762                                                 Diff_Cnt = Diff_Cnt - 1
 763                                         else:
 764                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 765
 766                                 SS_UpdateSize(src_file, dst_file)
 767                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 768
 769
 770         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 771         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 772
 773         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 774                 with open(VERBATIM_LIST, 'r') as F_News:
 775                         lines = set(F_News.read().splitlines())
 776                 for line in lines:
 777                         if not line in files_new:
 778                                 if os.path.exists(BASE_NEW+'/'+line):
 779                                         files_new.append(line)
 780                                         Verbatim_Cnt = Verbatim_Cnt+1
 781                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 782
 783
 784
 785         for elt in files_removed:
 786                 #if files are part of patches after renaming, we shd remove them as part of removed.
 787                 src_file = BASE_OLD+'/'+elt
 788                 if os.path.islink(src_file):
 789                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 790                 elif os.path.isdir(src_file):
 791                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 792                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 793                 else:
 794                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 795                 logging.debug(' File Deleted %s' % src_file)
 796                 Del_Cnt = Del_Cnt + 1
 797
 798         Dir_removed.sort(reverse=True)
 799         for elt in Dir_removed:
 800                 #if Dir is empty, add it to the removed list.
 801                 src_file = BASE_OLD+'/'+elt
 802                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 803                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 804                 if os.path.isdir(src_file):
 805                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 806                         Del_Cnt = Del_Cnt + 1
 807                         logging.debug(' Dir Deleted- %s' % src_file)
 808
 809
 810         for elt in files_new:
 811                 dst_file = BASE_NEW+'/'+elt
 812                 newfiles_dest_path = 'run/upgrade-sysroot/'
 813                 ensure_dir_exists(newfiles_dest_path)
 814                 if os.path.islink(dst_file):
 815                         patch = os.readlink(dst_file)
 816                         logging.debug(' File New Links %s' % elt)
 817                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 818                         #What if this is only a new sym link and folder already exists??? Should recheck
 819                         destpath = newfiles_dest_path + elt
 820                         if not os.path.exists(path_head(destpath)):
 821                                 os.makedirs(path_head(destpath))
 822                                 logging.info('New SymLink - Adding missing Dir')
 823                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 824                         Sym_New_Cnt = Sym_New_Cnt + 1
 825                 elif os.path.isdir(dst_file): # We create just empty directory here
 826                         destpath = newfiles_dest_path + elt
 827                         if not os.path.exists(destpath):
 828                                 os.makedirs(destpath)
 829                                 logging.debug(' File New Dir %s' % destpath)
 830                                 New_Cnt = New_Cnt + 1
 831                 else:
 832                         New_Cnt = New_Cnt + 1
 833                         destpath = newfiles_dest_path + elt
 834                         destdir = os.path.dirname(destpath)
 835                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 836
 837                         if not os.path.isdir(destdir):
 838                                 try:
 839                                         os.makedirs(destdir)
 840                                 except:
 841                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 842                                         raise
 843
 844                         try:
 845                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 846                                         shutil.copy2(dst_file, destpath)
 847                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 848                         except:
 849                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 850                                 raise
 851
 852         for elt in Dir_Added:
 853                 newfiles_dest_path = 'run/upgrade-sysroot/'
 854                 ensure_dir_exists(newfiles_dest_path)
 855                 destpath = newfiles_dest_path + elt
 856                 if not os.path.exists(destpath):
 857                         os.makedirs(destpath)
 858                         logging.debug(' DirList New Dir %s' % destpath)
 859                         New_Cnt = New_Cnt + 1
 860
 861         #Base directory should be system
 862         print 'Compressing New files'
 863         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 864                 WorkingDir = os.getcwd()
 865                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 866                 logging.info('Curr Working Dir - %s' % os.getcwd())
 867                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 868                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 869                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 870                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 871                 os.chdir(WorkingDir)
 872                 shutil.rmtree(NEW_FILES_PATH)
 873                 # use 7z a system.7z ./*
 874
 875         #logging.info('%d Dir to be removed' % len(Dir_removed))
 876         logging.info('%d files unchanged' % len(files_unchanged))
 877         logging.info('%d files files_renamed' % len(files_renamed))
 878         logging.info('%d files NEW' % len(files_new))
 879         logging.info('%d File attr' % len(File_Attibutes))
 880         logging.info('%d Sym attr' % len(Sym_Attibutes))
 881         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 882         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 883
 884         #There could be duplicates, TODO, can check before adding..
 885         ATTR_FILE_D = open(ATTR_FILE,'a+')
 886         for elt in File_Attibutes:
 887                 ATTR_FILE_D.write(elt)
 888         for elt in Sym_Attibutes:
 889                 ATTR_FILE_D.write(elt)
 890
 891         ATTR_FILE_D.close()
 892
 893         Partition_Doc_SymLinks.close()
 894         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 895         Partition_Doc.write(Partition_Read_SymLinks.read())
 896         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 897         Partition_Doc_SymLinks.close()
 898         Partition_Doc.close()
 899         os.remove(SymLinkDoc)
 900
 901         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 902                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 903                 logging.info('No Delta Generated for %s' % PART_NAME)
 904                 shutil.rmtree(OUT_DIR)
 905
 906
 907 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 908
 909         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 910         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 911         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 912         temp_apk = '../'+g_output_dir+'/'+b_apk
 913         Patch = 'Patch_'+b_apk
 914         ensure_dir_exists(Patch)
 915         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 916
 917         #Size issue on Device side?? shd check this
 918         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 919         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 920                 lines = set(f_new.read().splitlines())
 921                 for line in lines:
 922                         #print('Action ==> %s' % line)
 923                         #Action, Path, Patch = line.split('|')
 924                         Items = line.split('|')
 925                         Action = Items[0]
 926                         Path = Items[1]
 927                         ActualPath = a_folder+'/'+Path
 928                         PatchPath = Patch+'/'+Path
 929                         SrcPath = g_output_dir+'/'+path_leaf(Path)
 930                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 931                         if line[0] == 'c':
 932                                 patchName = g_output_dir+'/'+Items[2]
 933                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 934                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
 935                                 WorkingDir = os.getcwd()
 936                                 os.chdir(WorkingDir+"/"+"temp_a")
 937                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 938                                 os.chdir(WorkingDir)
 939                         elif line[0] == 's':
 940                                 WorkingDir = os.getcwd()
 941                                 os.chdir(WorkingDir+"/"+"temp_a")
 942                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 943                                 os.chdir(WorkingDir)
 944                         else:
 945                                 print('Apply_Container_Delta - Unknown Error')
 946         #print('Touch all files and set common attributes for DIFF generation')
 947         WorkingDir = os.getcwd()
 948         os.chdir(WorkingDir+"/"+Patch)
 949
 950         CONTAINER_DATE = '200011111111.11'
 951         CONTAINER_MODE = '0755'
 952         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
 953         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
 954         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
 955
 956         print 'Update Intermediate Archive'
 957         #subprocess.call(['zip','-ryX', b_apk, '*'])
 958         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
 959         os.chdir(WorkingDir)
 960         #print('Apply Path completed - Now create diff for this and place in patch folder')
 961         #print os.getcwd()
 962         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
 963         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
 964         patchLoc = '%s/%s' % (g_output_dir, patchName)
 965
 966         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
 967
 968         #Only on HOST... for testing
 969         if TEST_MODE == 'TRUE':
 970                 UpgradedName = '%s_Upgraded' % (b_apk)
 971                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
 972
 973         #This is file only with NEWS and empty diffs and same files.
 974         if TEST_MODE == 'FALSE':
 975                 os.remove(g_output_dir+'/'+b_apk)
 976                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
 977                 shutil.rmtree(Patch)
 978
 979 def IsSymlink(info):
 980   return (info.external_attr >> 16) == 0120777
 981
 982 def NewFiles(src, dest):
 983         print src,dest
 984         subprocess.call(['cp','-rp', src,dest])
 985     #try:
 986                 #shutil.copytree(src, dest)
 987     #except OSError as e:
 988         # If the error was caused because the source wasn't a directory
 989         #if e.errno == errno.ENOTDIR:
 990             #shutil.copy2(src, dest)
 991         #else:
 992             #print('Directory not copied. Error: %s' % e)
 993
 994 def measure_two_filediffs(src, dst):
 995         patchLoc = 'temp.patch'
 996         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
 997         result_size = os.path.getsize(patchLoc)
 998         os.remove(patchLoc)
 999         return result_size
1000
1001 def Get_Files(path):
1002         all_files = []
1003         all_dirs = []
1004
1005         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1006                 for directory in directories:
1007                         #DirName = os.path.join(root+'/',directory)
1008                         DirName = os.path.join(root,directory)
1009                         if os.path.islink(DirName):
1010                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1011                                 all_files.append(os.path.relpath(DirName, path))
1012                         elif not os.listdir(DirName):
1013                                 #print('*****Empty Directory******* -%s', DirName)
1014                                 #This should NOT be appended ??? Empty dir shd b considered
1015                                 all_dirs.append(os.path.relpath(DirName, path))
1016                         else:
1017                                 all_dirs.append(os.path.relpath(DirName, path))
1018                 for filename in filenames:
1019                         FileName = os.path.join(root,filename)
1020                         all_files.append(os.path.relpath(FileName, path))
1021
1022         all_files.sort()
1023         all_dirs.sort()
1024         return all_files, all_dirs
1025
1026
1027 USAGE_DOCSTRING = """
1028       Generate Delta using BASEOLD AND BASE NEW
1029           Attributes is optional
1030           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1031 """
1032
1033 def Usage(docstring):
1034   print docstring.rstrip("\n")
1035   print COMMON_DOCSTRING
1036
1037
1038
1039 if __name__ == '__main__':
1040         main()
1041