mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4
   5
   6 if sys.hexversion < 0x02040000:
   7   print >> sys.stderr, "Python 2.4 or newer is required."
   8   sys.exit(1)
   9
  10 import sys
  11 import os
  12 import filecmp
  13 import shutil
  14 import subprocess
  15 import re
  16 import ntpath
  17 import zipfile
  18 import datetime
  19 import hashlib
  20 import operator
  21 import locale
  22 import errno
  23 import logging
  24 import glob
  25 import apt
  26 import stat
  27 '''
  28 Diff two folders and create delta using SS_BSDIFF
  29 Will maintain same format of script that will be generated when we use diffutil
  30
  31 1. Create a list of files in each Base folders,
  32 2. These files will fall into one these below categories:
  33         1) Only in OLD - Should be deleted
  34         2) Only in NEW - Should be added or renamed accordingly
  35         3) File exists in both directories but contents are different - Create Diff.
  36         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  37         5) Duplicates in the list of Deletes and News
  38         6) Close matching diffs even though name changes across directories. (for matching extension)
  39         7) Clearing empty directories after Moves or diffs under Rename.
  40         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  41
  42 Current Case
  43 1. Given two folders, from list of REMOVED and NEW files find if there
  44 is version change and create diff between them
  45
  46 TODO
  47 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  48 Catching errors at all stages. SHOULD exit & return error in case of failure
  49 '''
  50
  51 def global_paths():
  52         global DIFF_UTIL
  53         global ZIPUTIL
  54         global NEW_FILES_PATH
  55         global NEW_FILES_FOLDER
  56         global NEW_FILES_ZIP_NAME
  57         global SYMLINK_TYPE
  58         global ATTR_DOC_EXT
  59         global SYMLINK_DOC_NAME
  60         global DIFF_PREFIX
  61         global DIFF_SUFFIX
  62         global SUPPORT_RENAME
  63         global NEW_PREFIX
  64         global DIFFPATCH_UTIL
  65         global SUPPORT_CONTAINERS
  66         global FULL_IMG
  67         global DELTA_IMG
  68         global DELTA_FS
  69         global EXTRA
  70         global COMMON_BIN_PATH
  71         global MEM_REQ
  72         global EMPTY
  73         global VERBATIM_LIST
  74         global MEM_FILE
  75
  76 COMMON_BIN_PATH = "../../common/bin/"
  77 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  78 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  79 ZIPUTIL = "p7zip "
  80 #ZIPUTIL = "7z a system.7z "
  81 NEW_FILES_PATH = "system"
  82 NEW_FILES_FOLDER  = "system"
  83 NEW_FILES_ZIP_NAME = "system.7z"
  84 SYMLINK_TYPE = "SYM"
  85 ATTR_DOC_EXT = "_attr.txt"
  86 SYMLINK_DOC_NAME = "_sym.txt"
  87 PART_DOC_EXT = ".txt"
  88 DIFF_PREFIX = "diff"
  89 DIFF_SUFFIX = ".delta"
  90 NEW_PREFIX = 'new'
  91 FULL_IMG = "FULL_IMG"
  92 DELTA_IMG = "DELTA_IMG"
  93 DELTA_FS = "DELTA_FS"
  94 EXTRA = "EXTRA"
  95 LOGFILE = "Delta.log"
  96 VERBATIM_LIST = "Verbatim_List.txt"
  97 EMPTY = ""
  98 MEM_REQ = 0
  99 MEM_FILE = "NULL"
 100
 101 SUPPORT_RENAME = "TRUE" #Use appropriate name
 102 SUPPORT_CONTAINERS = "FALSE"
 103 SUPPORT_DZIMAGE = "TRUE"
 104 SUPPORT_VERBATIM = "TRUE"
 105
 106 TEST_MODE = "FALSE"
 107
 108 def main():
 109         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 110         global AttributeFile
 111         global GenerateDiffAttr
 112         try:
 113
 114                 if len(sys.argv) < 5:
 115                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 116                 UPDATE_TYPE = sys.argv[1]
 117                 PART_NAME = sys.argv[2]  # lets make this also optional
 118
 119                 BASE_OLD = sys.argv[3]
 120                 BASE_NEW = sys.argv[4]
 121                 OUT_DIR = sys.argv[5]
 122                 ATTR_OLD = EMPTY
 123                 ATTR_NEW = EMPTY
 124                 UPDATE_CFG_PATH = EMPTY
 125                 GenerateDiffAttr = "FALSE"
 126                 if UPDATE_TYPE == DELTA_FS:
 127                         #instead of arguments check it in outdirectory ?
 128                         if len(sys.argv) == 9:
 129                                 ATTR_OLD = sys.argv[6]
 130                                 ATTR_NEW = sys.argv[7]
 131                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 132                                 GenerateDiffAttr = "TRUE"
 133
 134                 elif UPDATE_TYPE == DELTA_IMG or UPDATE_TYPE == FULL_IMG:
 135                         if len(sys.argv) == 7:
 136                                 #Use path in better way
 137                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 138
 139                 global DIFF_UTIL
 140                 global DIFFPATCH_UTIL
 141                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 142                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 143                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 144                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 145                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 146                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 147                                 sys.exit(1)
 148
 149                 start = datetime.datetime.now().time()
 150                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 151                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 152
 153                 ensure_dir_exists(OUT_DIR)
 154                 if GenerateDiffAttr == "TRUE":
 155                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 156                                 print >> sys.stderr, "Attributes missing -- ABORT"
 157                                 sys.exit(1)
 158
 159
 160                 # Should check if APT is supported on other linux flavours
 161                 cache = apt.Cache()
 162                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 163                         logging.info ('Basic utils installed')
 164                 else:
 165                         print >> sys.stderr, "Basic utils missing -- ABORT"
 166                         sys.exit(1)
 167
 168                 if UPDATE_TYPE == FULL_IMG:
 169                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 170                 elif UPDATE_TYPE == DELTA_IMG:
 171                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 172                 elif UPDATE_TYPE == DELTA_FS:
 173                         AttributeFile = ATTR_NEW
 174                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 175                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 176                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 177                         New_files, New_dirs = Get_Files(BASE_NEW)
 178                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 179
 180                         if not UPDATE_CFG_PATH == EMPTY:
 181                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 182
 183
 184                 elif UPDATE_TYPE == EXTRA:
 185                         print('UPDATE_TYPE ---- EXTRA')
 186                 else:
 187                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 188
 189                 if GenerateDiffAttr == "TRUE":
 190                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 191                                 os.remove(ATTR_OLD)
 192                                 os.remove(ATTR_NEW)
 193                 end = datetime.datetime.now().time()
 194
 195                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 196                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 197                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 198                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 199
 200         except:
 201                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 202                 raise
 203
 204
 205 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 206         f = open(UPDATE_CFG_PATH, 'r')
 207         lines = f.readlines()
 208         f.close()
 209         f = open(UPDATE_CFG_PATH, 'w')
 210         for line in lines:
 211                 ConfigItems = line.split()
 212                 if ConfigItems[0] == DELTA_BIN:
 213                         DELTA = ConfigItems[1]
 214                         logging.info ('Updating %s config' % DELTA_BIN)
 215                         line = line.rstrip('\n')
 216                         Value = MEM_REQ
 217                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 218                         f.write(line)
 219                 else:
 220                         f.write(line)
 221         f.close()
 222
 223 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 224         #for sizes
 225
 226         ZIMAGE_SCRIPT = COMMON_BIN_PATH+'./dzImagescript.sh'
 227         ZIMAGE_OLD = BASE_OLD+'_unpacked'
 228         ZIMAGE_NEW = BASE_NEW+'_unpacked'
 229         DZIMAGE_HEADER = 'UnpackdzImage'
 230         DZIMAGE_SEP = ':'
 231
 232         oldsize_d= os.path.getsize(BASE_OLD)
 233         newsize_d= os.path.getsize(BASE_NEW)
 234         SHA_BIN_DEST= hash_file(BASE_NEW)
 235         SHA_BIN_BASE=hash_file(BASE_OLD)
 236
 237         #incase UPDATE CFG is empty
 238         DELTA = DELTA_BIN
 239         SS_UpdateSize(BASE_OLD, BASE_NEW)
 240         #Should throw error if PART NAME NOT found??
 241         if not UPDATE_CFG_PATH == EMPTY:
 242                 f = open(UPDATE_CFG_PATH, 'r')
 243                 lines = f.readlines()
 244                 f.close()
 245                 f = open(UPDATE_CFG_PATH, 'w')
 246                 for line in lines:
 247                         ConfigItems = line.split()
 248                         if ConfigItems[0] == DELTA_BIN:
 249                                 logging.info ('Updating %s config' % DELTA_BIN)
 250                                 DELTA = ConfigItems[1]
 251                                 line = line.rstrip('\n')
 252                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 253                                 f.write(line)
 254                         else:
 255                                 f.write(line)
 256                 f.close()
 257
 258         #Any validation checks required?
 259         if (DELTA_BIN == "zImage" or DELTA_BIN == "dzImage" or DELTA_BIN == "KERNEL" or DELTA_BIN == "BOOT") and SUPPORT_DZIMAGE == "TRUE":
 260
 261                 #Unpack Old and New Images for creating delta
 262                 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_OLD])
 263                 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_NEW])
 264
 265                 DeltaFiles = []
 266                 Old_files, Old_dirs = Get_Files(ZIMAGE_OLD)
 267                 New_files, New_dirs = Get_Files(ZIMAGE_NEW)
 268
 269                 patchLoc = '%s/%s_temp' % (OUT_DIR, DELTA_BIN)
 270                 ensure_dir_exists(patchLoc)
 271
 272                 for elt in New_files:
 273                         if elt in Old_files:
 274                                 src_file = ZIMAGE_OLD+'/'+elt
 275                                 dst_file = ZIMAGE_NEW+'/'+elt
 276                                 if not filecmp.cmp(src_file, dst_file):
 277                                         patch = '%s/%s' % (patchLoc,elt)
 278                                         DeltaFiles.append(patch)
 279                                         subprocess.call([DIFF_UTIL,src_file,dst_file,patch])
 280                                         logging.info('Make dz Image %s <--> %s ==> %s %s' % (src_file, dst_file , DELTA_BIN, patch))
 281
 282                 #Append all delta files to make image.delta
 283
 284                 #HEADER FORMAT MAGICNAME:FILECOUNT:[FILENAME:FILESIZE:][FILECONTENT/S]
 285                 HeaderStr = DZIMAGE_HEADER+DZIMAGE_SEP+'%d' % len(DeltaFiles)
 286                 HeaderStr = HeaderStr+DZIMAGE_SEP
 287
 288                 with open(OUT_DIR+'/'+DELTA, 'w') as DeltaFile:
 289                         for fname in DeltaFiles:
 290                                 DeltaSize = os.path.getsize(fname)
 291                                 HeaderStr = HeaderStr+path_leaf(fname)+DZIMAGE_SEP+'%d' % DeltaSize
 292                                 HeaderStr = HeaderStr+DZIMAGE_SEP
 293                         #Using 128 bytes as max Header.
 294                         logging.info('zImage Header - %s' % HeaderStr.ljust(128,'0'))
 295                         DeltaFile.write(HeaderStr.ljust(128,'0'))
 296                         for fname in DeltaFiles:
 297                                 with open(fname) as infile:
 298                                         DeltaFile.write(infile.read())
 299                                         infile.close()
 300
 301                 DeltaFile.close()
 302                 shutil.rmtree(patchLoc)
 303                 shutil.rmtree(ZIMAGE_OLD)
 304                 shutil.rmtree(ZIMAGE_NEW)
 305                 #Do we need to incorprate Max memory required for backup??
 306
 307         else:
 308                 patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 309                 logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 310                 subprocess.call([DIFF_UTIL,BASE_OLD,BASE_NEW,patchLoc])
 311
 312
 313
 314 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 315         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 316         oldsize_d= os.path.getsize(BASE_OLD)
 317         newsize_d= os.path.getsize(BASE_NEW)
 318         SHA_BIN_DEST= hash_file(BASE_NEW)
 319         SHA_BIN_BASE=hash_file(BASE_OLD)
 320         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 321         SS_UpdateSize(BASE_OLD, BASE_NEW)
 322
 323         if not UPDATE_CFG_PATH == EMPTY:
 324                 f = open(UPDATE_CFG_PATH, 'r')
 325                 lines = f.readlines()
 326                 f.close()
 327                 f = open(UPDATE_CFG_PATH, 'w')
 328                 for line in lines:
 329                         ConfigItems = line.split()
 330                         if ConfigItems[0] == DELTA_BIN:
 331                                 logging.info ('Updating %s config' % DELTA_BIN)
 332                                 DELTA = ConfigItems[1]
 333                                 line = line.rstrip('\n')
 334                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 335                                 f.write(line)
 336                         else:
 337                                 f.write(line)
 338                 f.close()
 339
 340 def zipdir(path, zip):
 341     for root, dirs, files in os.walk(path):
 342         for file in files:
 343             zip.write(os.path.join(root, file))
 344
 345 def ensure_dir_exists(path):
 346         if not os.path.exists(path):
 347                 os.makedirs(path)
 348                 #shutil.rmtree(path)
 349         #os.makedirs(path)
 350
 351
 352 def path_leaf(path):
 353     head, tail = ntpath.split(path) #This is for windows?? Recheck
 354     return tail
 355
 356 def path_head(path):
 357     head, tail = ntpath.split(path)
 358     return head
 359
 360 def difflines(list1, list2):
 361     c = set(list1).union(set(list2))
 362     d = set(list1).intersection(set(list2))
 363     return list(c-d)
 364
 365 #Creating Diff between OLD and NEW attribute files v12
 366 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 367         if GenerateDiffAttr == "FALSE":
 368                 return
 369         with open(ATTR_OLD, 'r') as f_old:
 370                 lines1 = set(f_old.read().splitlines())
 371
 372         with open(ATTR_NEW, 'r') as f_new:
 373                 lines2 = set(f_new.read().splitlines())
 374
 375         lines = difflines(lines2, lines1)
 376         with open(ATTR_FILE, 'w+') as file_out:
 377                 for line in lines:
 378                         if line not in lines1:
 379                                 logging.info('Diff_AttrFiles - %s' % line)
 380                                 file_out.write(line+'\n')
 381
 382         f_new.close()
 383         f_old.close()
 384         file_out.close()
 385
 386
 387
 388 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 389         #Full File Path should MATCH
 390         if GenerateDiffAttr == "FALSE":
 391                 return
 392         FilePath = '"/'+RequestedPath+'"'
 393         #print ('FilePath - %s'% (FilePath))
 394         with open(AttributeFile) as f:
 395                 for line in f:
 396                         if FilePath in line:
 397                                 if Type == SYMLINK_TYPE:
 398                                         Sym_Attibutes.append(line)
 399                                 else:
 400                                         File_Attibutes.append(line)
 401
 402
 403 '''This function returns the SHA-1 hash of the file passed into it'''
 404 def hash_file(filename):
 405
 406    # make a hash object
 407    h = hashlib.sha1()
 408
 409    # open file for reading in binary mode
 410    with open(filename,'rb') as file:
 411        # loop till the end of the file
 412        chunk = 0
 413        while chunk != b'':
 414            # read only 1024 bytes at a time
 415            chunk = file.read(1024*1024)
 416            h.update(chunk)
 417
 418    # return the hex representation of digest
 419    return h.hexdigest()
 420
 421 def find_dupes_dir(BASE_OLD, BASE_NEW):
 422         dups = {}
 423         fdupes = {}
 424         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 425         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 426         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 427                 #print('Scanning %s...' % rootbase)
 428                 for filename in fileListB:
 429                         path = os.path.join(rootbase, filename)
 430                         if os.path.islink(path):
 431                                 continue
 432                         # Calculate hash
 433                         file_hash = hash_file(path)
 434                         dups[file_hash] = path
 435
 436         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 437                 #print('Scanning %s...' % roottarget)
 438                 for filename in fileListT:
 439                         # Get the path to the file
 440                         path = os.path.join(roottarget, filename)
 441                         if os.path.islink(path):
 442                                 continue
 443                         # Calculate hash
 444                         file_hash = hash_file(path)
 445                         # Add or append the file path
 446                         if file_hash in dups:
 447                                 BaseStr = dups.get(file_hash)
 448                                 Baseloc = path.find('/')
 449                                 TarLoc = BaseStr.find('/')
 450                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 451                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 452                                         fdupes[path] = BaseStr
 453         logging.info('Total Duplicate files %d' % (len(fdupes)))
 454         return fdupes
 455
 456
 457 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 458         dups = {}
 459         fdupes = {}
 460         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 461
 462         for filename in fileListB:
 463                 Src_File = BASE_OLD+'/'+filename
 464                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 465                         continue
 466                 # Calculate hash
 467                 file_hash = hash_file(Src_File)
 468                 dups[file_hash] = Src_File
 469
 470
 471         for filename in fileListT:
 472                 Dest_File = BASE_NEW+'/'+filename
 473                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 474                         continue
 475                 # Calculate hash
 476                 file_hash = hash_file(Dest_File)
 477                 if file_hash in dups:
 478                         BaseStr = dups.get(file_hash)
 479                         Baseloc = BaseStr.find('/')
 480                         if not BaseStr[Baseloc:] == filename:
 481                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 482                                 fdupes[BaseStr] = filename
 483
 484         logging.info('Total Duplicate files %d' % (len(fdupes)))
 485         return fdupes
 486
 487 def SS_UpdateSize(src_file, dst_file):
 488         global MEM_REQ
 489         global MEM_FILE
 490         oldsize_d= os.path.getsize(src_file)
 491         newsize_d= os.path.getsize(dst_file)
 492         if oldsize_d >= newsize_d:
 493                 Max = newsize_d
 494         else:
 495                 Max = oldsize_d
 496         if MEM_REQ < Max:
 497                 MEM_REQ = Max
 498                 MEM_FILE = dst_file
 499
 500
 501
 502 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 503         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 504         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 505
 506         # First let's fill up these categories
 507         files_new = []
 508         files_removed = []
 509         Dir_removed = []
 510         Dir_Added = []
 511         files_changed = []
 512         files_unchanged = []
 513         files_renamed = []
 514         File_Attibutes = []
 515         Sym_Attibutes = []
 516
 517         files_Del_List = {}
 518         files_New_List = {}
 519         MyDict_Patches = {}
 520
 521
 522
 523         PWD = os.getcwd()
 524
 525         # Generate NEW List
 526         for elt in New_files:
 527                 if elt not in Old_files:
 528                         files_new.append(elt)
 529                         logging.info('New files %s' % elt)
 530
 531         # Generate Delete List
 532         for elt in Old_files:
 533                 if elt not in New_files:
 534                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 535                         files_removed.append(elt)
 536                         logging.info('Old files %s' % elt)
 537
 538
 539         for elt in Old_dirs:
 540                 #print('List of Old Dirs %s' % elt)
 541                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 542                 if elt not in New_dirs:
 543                         Dir_removed.append(elt)
 544                         logging.info('Old Dirs %s' % elt+'/')
 545
 546         for elt in New_dirs:
 547                 if elt not in Old_dirs:
 548                         Dir_Added.append(elt)
 549                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 550
 551         # What files have changed contents but not name/path?
 552         for elt in New_files:
 553                 if elt in Old_files:
 554                         #Both are symbolic linkes and they differ
 555                         src_file = BASE_OLD+'/'+elt
 556                         dst_file = BASE_NEW+'/'+elt
 557                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 558                         if os.path.islink(src_file) and os.path.islink(dst_file):
 559                                 if not os.readlink(src_file) == os.readlink(dst_file):
 560                                         files_changed.append(elt)
 561                                         #print('%d Sym link files changed' % len(files_changed))
 562                                         logging.info('Sym links Changed - %s' % elt)
 563                                 else:
 564                                         files_unchanged.append(elt)
 565                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 566                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 567                                 if not filecmp.cmp(src_file, dst_file):
 568                                         files_changed.append(elt)
 569                                         #print('%d Normal files changed' % len(files_changed))
 570                                         #print('Files Changed - %s' % elt)
 571                                 else:
 572                                         files_unchanged.append(elt)
 573                         #File types differ between BASE and TARGET
 574                         else:
 575                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 576                                 #Both file types have changed and they differ
 577                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 578                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 579                                 files_removed.append(elt)
 580                                 files_new.append(elt)
 581
 582
 583         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 584         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 585
 586         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 587         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 588                 with open(VERBATIM_LIST, 'r') as F_News:
 589                         lines = set(F_News.read().splitlines())
 590                 for line in lines:
 591                         if line in files_changed:
 592                                 files_changed.remove(line)
 593                                 files_removed.append(line)
 594                         if line in files_new:
 595                                 files_new.remove(line)
 596
 597         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 598         if SUPPORT_RENAME == "TRUE":
 599                 for elt in files_removed:
 600                         if os.path.isfile(BASE_OLD+'/'+elt):
 601                                 FileName = path_leaf(elt)
 602                                 entries = re.split('[0-9]' , FileName)
 603                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 604                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 605                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 606                                 if len(entries[0]) > 0:
 607                                         files_Del_List.update({entries[0]: elt})
 608
 609                 for elt in files_new:
 610                         if os.path.isfile(BASE_NEW+'/'+elt):
 611                                 FileName = path_leaf(elt)
 612                                 entries = re.split('[0-9]' , FileName)
 613                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 614                                 if len(entries[0]) > 0:
 615                                         files_New_List.update({entries[0]: elt})
 616
 617                 for key, value in files_Del_List.iteritems():
 618                         #print('Key value pair -%s -%s' % (key, value))
 619                         if key in files_New_List:
 620                                 # this file is the same name in both!
 621                                 src_file = BASE_OLD+'/'+value
 622                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 623                                 olddirpath = path_head(files_New_List[key])
 624                                 newdirpath = path_head(value)
 625                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 626                                         logging.debug('Cannot diff as one of them is Symlink')
 627                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 628                                         logging.debug('Cannot diff as one of them is dir')
 629                                 else:
 630                                         #Pick the best diff of same type and diff names
 631                                         files_renamed.append([files_New_List[key], value])
 632                                         files_removed.remove(value)
 633                                         files_new.remove(files_New_List[key])
 634
 635         '''
 636         Patch Section
 637                 Partition.txt contains Protocol for UPI
 638                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 639         '''
 640         Sym_Diff_Cnt = 0
 641         Sym_New_Cnt = 0;
 642         Del_Cnt = 0
 643         New_Cnt = 0
 644         Diff_Cnt = 0
 645         Move_Cnt = 0
 646         Verbatim_Cnt = 0
 647         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 648         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 649         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 650
 651         print("writing diff'ed changed files...")
 652         for elt in files_changed:
 653                 dst_file = BASE_NEW+'/'+elt
 654                 src_file = BASE_OLD+'/'+elt
 655                 #Both files are symbolic links and they differ
 656                 if os.path.islink(dst_file) and os.path.islink(src_file):
 657                 #Both are symlinks and they differ
 658                         logging.debug(' File Changed is Link %s ' % dst_file)
 659                         patch = os.readlink(dst_file)
 660                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 661                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 662                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 663                 #Both are NORMAL files and they differ
 664                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 665                         #Both are files and they differ
 666                         Diff_Cnt = Diff_Cnt + 1
 667                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 668                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 669                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 670                         SS_UpdateSize(src_file, dst_file)
 671
 672                         FORMAT = "REG"
 673                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 674                         if ret is not 0:
 675                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 676                                 files_new.append(elt)
 677                                 Diff_Cnt = Diff_Cnt - 1
 678                         else:
 679                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 680
 681                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 682                 #Both differ but they are of diff types
 683                 else:
 684                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 685                         files_removed.append(elt)
 686                         files_new.append(elt)
 687
 688         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 689         for oldpath, newpath in fdupes.iteritems():
 690                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 691
 692         for elt in files_removed:
 693                 src_file = BASE_OLD+'/'+elt
 694                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 695                 if src_file in fdupes.keys():
 696                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 697                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 698                         Move_Cnt = Move_Cnt + 1
 699                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 700                         files_removed.remove(elt)
 701                         files_new.remove(fdupes[src_file])
 702
 703         #Should be placed after removing duplicates, else they will be filtered here.
 704         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 705         #       First Step: Sort & Filter out unwanted files
 706         # Minimum condition used is,
 707         #       1. File name should match 70%
 708         #       2. Extensions should be same
 709         #       3. File name length shd b greater than 3 char
 710         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 711         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 712
 713         Base_DelList = files_removed[:]
 714         Base_NewList = files_new[:]
 715         DelList = sorted(Base_DelList, key=path_leaf)
 716         NewList = sorted(Base_NewList, key=path_leaf)
 717         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 718
 719         Filter1 = []
 720         Filter2 = []
 721         #Remove unwanted items which we cant make diff with for rename logic
 722         for file in DelList:
 723                 if os.path.islink(BASE_OLD+'/'+file):
 724                         continue
 725                 elif os.path.isdir(BASE_OLD+'/'+file):
 726                         continue
 727                 else:
 728                         Filter1.append(file)
 729                         #logging.debug('Sorted del list - %s' % (file))
 730
 731         DelList = Filter1
 732
 733         for file in NewList:
 734                 if os.path.islink(BASE_NEW+'/'+file):
 735                         continue
 736                 elif os.path.isdir(BASE_NEW+'/'+file):
 737                         continue
 738                 elif len(path_leaf(file)) <= 3:
 739                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 740                         continue
 741                 else:
 742                         Filter2.append(file)
 743
 744         NewList = Filter2
 745
 746         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 747
 748         for new_file in NewList:
 749                 R_Flag = 'FALSE';
 750                 DirPathNew = path_head(new_file)
 751                 FileNameNew = path_leaf(new_file)
 752                 DiffSize = 0
 753                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 754                 New_fs = winning_patch_sz
 755                 winning_file = ''
 756
 757                 for del_file in DelList:
 758                         FileNameOld = path_leaf(del_file)
 759                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 760                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 761                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 762                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 763                                 v1 = abs(New_fs-Del_fs)
 764                                 v2 = (New_fs+Del_fs)/2
 765                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 766                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 767                                         continue
 768                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 769                                 R_Flag = 'TRUE';
 770                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 771                                 if (DiffSize < 0.8 * winning_patch_sz):
 772                                         winning_patch_sz = DiffSize
 773                                         winning_file = del_file
 774                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 775                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 776                                 break;
 777                 if len(winning_file) > 0:
 778                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 779                         files_renamed.append([new_file, winning_file])
 780                         DelList.remove(winning_file)
 781                         files_removed.remove(winning_file)
 782                         files_new.remove(new_file)
 783
 784         #********************** Files should NOT be deleted for any such renames ***********************
 785
 786         if SUPPORT_RENAME == "TRUE":
 787                 for elt in files_renamed:
 788                         src_file = BASE_OLD+'/'+elt[1]
 789                         dst_file = BASE_NEW+'/'+elt[0]
 790                         Diff_Cnt = Diff_Cnt + 1
 791                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 792                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 793                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 794                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 795                         # Should be careful of renaming files??
 796                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 797                         # What if folder path has numerics??
 798
 799                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 800                                 #This case never occurs??
 801                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 802                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 803                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 804                                 if filecmp.cmp(src_file, dst_file):
 805                                         Move_Cnt = Move_Cnt + 1
 806                                         Diff_Cnt = Diff_Cnt - 1
 807                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 808                                 else:
 809                                         FORMAT = "REG"
 810                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 811                                         if ret is not 0:
 812                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 813                                                 files_new.append(elt)
 814                                                 Diff_Cnt = Diff_Cnt - 1
 815                                         else:
 816                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 817
 818                                 SS_UpdateSize(src_file, dst_file)
 819                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 820
 821
 822         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 823         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 824
 825         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 826                 with open(VERBATIM_LIST, 'r') as F_News:
 827                         lines = set(F_News.read().splitlines())
 828                 for line in lines:
 829                         if not line in files_new:
 830                                 if os.path.exists(BASE_NEW+'/'+line):
 831                                         files_new.append(line)
 832                                         Verbatim_Cnt = Verbatim_Cnt+1
 833                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 834
 835
 836
 837         for elt in files_removed:
 838                 #if files are part of patches after renaming, we shd remove them as part of removed.
 839                 src_file = BASE_OLD+'/'+elt
 840                 if os.path.islink(src_file):
 841                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 842                 elif os.path.isdir(src_file):
 843                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 844                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 845                 else:
 846                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 847                 logging.debug(' File Deleted %s' % src_file)
 848                 Del_Cnt = Del_Cnt + 1
 849
 850         Dir_removed.sort(reverse=True)
 851         for elt in Dir_removed:
 852                 #if Dir is empty, add it to the removed list.
 853                 src_file = BASE_OLD+'/'+elt
 854                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 855                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 856                 if os.path.isdir(src_file):
 857                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 858                         Del_Cnt = Del_Cnt + 1
 859                         logging.debug(' Dir Deleted- %s' % src_file)
 860
 861
 862         for elt in files_new:
 863                 dst_file = BASE_NEW+'/'+elt
 864                 newfiles_dest_path = 'system/'
 865                 ensure_dir_exists(newfiles_dest_path)
 866                 if os.path.islink(dst_file):
 867                         patch = os.readlink(dst_file)
 868                         logging.debug(' File New Links %s' % elt)
 869                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 870                         #What if this is only a new sym link and folder already exists??? Should recheck
 871                         destpath = newfiles_dest_path + elt
 872                         if not os.path.exists(path_head(destpath)):
 873                                 os.makedirs(path_head(destpath))
 874                                 logging.info('New SymLink - Adding missing Dir')
 875                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 876                         Sym_New_Cnt = Sym_New_Cnt + 1
 877                 elif os.path.isdir(dst_file): # We create just empty directory here
 878                         destpath = newfiles_dest_path + elt
 879                         if not os.path.exists(destpath):
 880                                 os.makedirs(destpath)
 881                                 logging.debug(' File New Dir %s' % destpath)
 882                                 New_Cnt = New_Cnt + 1
 883                 else:
 884                         New_Cnt = New_Cnt + 1
 885                         #newfiles_dest_path = OUT_DIR + '/system/'
 886                         destpath = newfiles_dest_path + elt
 887                         destdir = os.path.dirname(destpath)
 888                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 889
 890                         if not os.path.isdir(destdir):
 891                                 try:
 892                                         os.makedirs(destdir)
 893                                 except:
 894                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 895                                         raise
 896
 897                         try:
 898                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 899                                         shutil.copy2(dst_file, destpath)
 900                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 901                         except:
 902                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 903                                 raise
 904
 905         for elt in Dir_Added:
 906                 newfiles_dest_path = 'system/'
 907                 ensure_dir_exists(newfiles_dest_path)
 908                 destpath = newfiles_dest_path + elt
 909                 if not os.path.exists(destpath):
 910                         os.makedirs(destpath)
 911                         logging.debug(' DirList New Dir %s' % destpath)
 912                         New_Cnt = New_Cnt + 1
 913
 914         #Base directory should be system
 915         print 'Compressing New files'
 916         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 917                 WorkingDir = os.getcwd()
 918                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 919                 logging.info('Curr Working Dir - %s' % os.getcwd())
 920                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 921                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 922                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 923                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 924                 os.chdir(WorkingDir)
 925                 shutil.rmtree(NEW_FILES_PATH)
 926                 # use 7z a system.7z ./*
 927
 928         #logging.info('%d Dir to be removed' % len(Dir_removed))
 929         logging.info('%d files unchanged' % len(files_unchanged))
 930         logging.info('%d files files_renamed' % len(files_renamed))
 931         logging.info('%d files NEW' % len(files_new))
 932         logging.info('%d File attr' % len(File_Attibutes))
 933         logging.info('%d Sym attr' % len(Sym_Attibutes))
 934         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 935         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 936
 937         #There could be duplicates, TODO, can check before adding..
 938         ATTR_FILE_D = open(ATTR_FILE,'a+')
 939         for elt in File_Attibutes:
 940                 ATTR_FILE_D.write(elt)
 941         for elt in Sym_Attibutes:
 942                 ATTR_FILE_D.write(elt)
 943
 944         ATTR_FILE_D.close()
 945
 946         Partition_Doc_SymLinks.close()
 947         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 948         Partition_Doc.write(Partition_Read_SymLinks.read())
 949         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 950         Partition_Doc_SymLinks.close()
 951         Partition_Doc.close()
 952         os.remove(SymLinkDoc)
 953
 954         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 955                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 956                 logging.info('No Delta Generated for %s' % PART_NAME)
 957                 shutil.rmtree(OUT_DIR)
 958
 959
 960 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 961
 962         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 963         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 964         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 965         temp_apk = '../'+g_output_dir+'/'+b_apk
 966         Patch = 'Patch_'+b_apk
 967         ensure_dir_exists(Patch)
 968         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 969
 970         #Size issue on Device side?? shd check this
 971         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 972         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 973                 lines = set(f_new.read().splitlines())
 974                 for line in lines:
 975                         #print('Action ==> %s' % line)
 976                         #Action, Path, Patch = line.split('|')
 977                         Items = line.split('|')
 978                         Action = Items[0]
 979                         Path = Items[1]
 980                         ActualPath = a_folder+'/'+Path
 981                         PatchPath = Patch+'/'+Path
 982                         SrcPath = g_output_dir+'/'+path_leaf(Path)
 983                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 984                         if line[0] == 'c':
 985                                 patchName = g_output_dir+'/'+Items[2]
 986                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 987                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
 988                                 WorkingDir = os.getcwd()
 989                                 os.chdir(WorkingDir+"/"+"temp_a")
 990                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 991                                 os.chdir(WorkingDir)
 992                         elif line[0] == 's':
 993                                 WorkingDir = os.getcwd()
 994                                 os.chdir(WorkingDir+"/"+"temp_a")
 995                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 996                                 os.chdir(WorkingDir)
 997                         else:
 998                                 print('Apply_Container_Delta - Unknown Error')
 999         #print('Touch all files and set common attributes for DIFF generation')
1000         WorkingDir = os.getcwd()
1001         os.chdir(WorkingDir+"/"+Patch)
1002
1003         CONTAINER_DATE = '200011111111.11'
1004         CONTAINER_MODE = '0755'
1005         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
1006         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
1007         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
1008
1009         print 'Update Intermediate Archive'
1010         #subprocess.call(['zip','-ryX', b_apk, '*'])
1011         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
1012         os.chdir(WorkingDir)
1013         #print('Apply Path completed - Now create diff for this and place in patch folder')
1014         #print os.getcwd()
1015         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
1016         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
1017         patchLoc = '%s/%s' % (g_output_dir, patchName)
1018
1019         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
1020
1021         #Only on HOST... for testing
1022         if TEST_MODE == 'TRUE':
1023                 UpgradedName = '%s_Upgraded' % (b_apk)
1024                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
1025
1026         #This is file only with NEWS and empty diffs and same files.
1027         if TEST_MODE == 'FALSE':
1028                 os.remove(g_output_dir+'/'+b_apk)
1029                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
1030                 shutil.rmtree(Patch)
1031
1032 def IsSymlink(info):
1033   return (info.external_attr >> 16) == 0120777
1034
1035 def NewFiles(src, dest):
1036         print src,dest
1037         subprocess.call(['cp','-rp', src,dest])
1038     #try:
1039                 #shutil.copytree(src, dest)
1040     #except OSError as e:
1041         # If the error was caused because the source wasn't a directory
1042         #if e.errno == errno.ENOTDIR:
1043             #shutil.copy2(src, dest)
1044         #else:
1045             #print('Directory not copied. Error: %s' % e)
1046
1047 def measure_two_filediffs(src, dst):
1048         patchLoc = 'temp.patch'
1049         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
1050         result_size = os.path.getsize(patchLoc)
1051         os.remove(patchLoc)
1052         return result_size
1053
1054 def Get_Files(path):
1055         all_files = []
1056         all_dirs = []
1057
1058         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1059                 for directory in directories:
1060                         #DirName = os.path.join(root+'/',directory)
1061                         DirName = os.path.join(root,directory)
1062                         if os.path.islink(DirName):
1063                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1064                                 all_files.append(os.path.relpath(DirName, path))
1065                         elif not os.listdir(DirName):
1066                                 #print('*****Empty Directory******* -%s', DirName)
1067                                 #This should NOT be appended ??? Empty dir shd b considered
1068                                 all_dirs.append(os.path.relpath(DirName, path))
1069                         else:
1070                                 all_dirs.append(os.path.relpath(DirName, path))
1071                 for filename in filenames:
1072                         FileName = os.path.join(root,filename)
1073                         all_files.append(os.path.relpath(FileName, path))
1074
1075         all_files.sort()
1076         all_dirs.sort()
1077         return all_files, all_dirs
1078
1079
1080 USAGE_DOCSTRING = """
1081       Generate Delta using BASEOLD AND BASE NEW
1082           Attributes is optional
1083           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1084 """
1085
1086 def Usage(docstring):
1087   print docstring.rstrip("\n")
1088   print COMMON_DOCSTRING
1089
1090
1091
1092 if __name__ == '__main__':
1093         main()
1094