mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49 def global_paths():
  50         global DIFF_UTIL
  51         global ZIPUTIL
  52         global NEW_FILES_PATH
  53         global NEW_FILES_FOLDER
  54         global NEW_FILES_ZIP_NAME
  55         global SYMLINK_TYPE
  56         global ATTR_DOC_EXT
  57         global SYMLINK_DOC_NAME
  58         global DIFF_PREFIX
  59         global DIFF_SUFFIX
  60         global SUPPORT_RENAME
  61         global NEW_PREFIX
  62         global DIFFPATCH_UTIL
  63         global SUPPORT_CONTAINERS
  64         global FULL_IMG
  65         global DELTA_IMG
  66         global DELTA_FS
  67         global EXTRA
  68         global COMMON_BIN_PATH
  69         global MEM_REQ
  70         global EMPTY
  71         global VERBATIM_LIST
  72         global MEM_FILE
  73
  74 COMMON_BIN_PATH = "../../common/bin/"
  75 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  76 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  77 ZIPUTIL = "p7zip "
  78 #ZIPUTIL = "7z a system.7z "
  79 NEW_FILES_PATH = "system"
  80 NEW_FILES_FOLDER  = "system"
  81 NEW_FILES_ZIP_NAME = "system.7z"
  82 SYMLINK_TYPE = "SYM"
  83 ATTR_DOC_EXT = "_attr.txt"
  84 SYMLINK_DOC_NAME = "_sym.txt"
  85 PART_DOC_EXT = ".txt"
  86 DIFF_PREFIX = "diff"
  87 DIFF_SUFFIX = ".delta"
  88 NEW_PREFIX = 'new'
  89 FULL_IMG = "FULL_IMG"
  90 DELTA_IMG = "DELTA_IMG"
  91 DELTA_FS = "DELTA_FS"
  92 EXTRA = "EXTRA"
  93 PRE_UA = "PRE_UA"
  94 LOGFILE = "Delta.log"
  95 VERBATIM_LIST = "Verbatim_List.txt"
  96 EMPTY = ""
  97 MEM_REQ = 0
  98 MEM_FILE = "NULL"
  99
 100 SUPPORT_RENAME = "TRUE" #Use appropriate name
 101 SUPPORT_CONTAINERS = "FALSE"
 102 SUPPORT_DZIMAGE = "TRUE"
 103 SUPPORT_VERBATIM = "TRUE"
 104
 105 TEST_MODE = "FALSE"
 106
 107 def main():
 108         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 109         global AttributeFile
 110         global GenerateDiffAttr
 111         try:
 112
 113                 if len(sys.argv) < 5:
 114                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 115                 UPDATE_TYPE = sys.argv[1]
 116                 PART_NAME = sys.argv[2]  # lets make this also optional
 117
 118                 BASE_OLD = sys.argv[3]
 119                 BASE_NEW = sys.argv[4]
 120                 OUT_DIR = sys.argv[5]
 121                 ATTR_OLD = EMPTY
 122                 ATTR_NEW = EMPTY
 123                 UPDATE_CFG_PATH = EMPTY
 124                 GenerateDiffAttr = "FALSE"
 125                 if UPDATE_TYPE == DELTA_FS:
 126                         #instead of arguments check it in outdirectory ?
 127                         if len(sys.argv) == 9:
 128                                 ATTR_OLD = sys.argv[6]
 129                                 ATTR_NEW = sys.argv[7]
 130                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 131                                 GenerateDiffAttr = "TRUE"
 132
 133                 elif UPDATE_TYPE == DELTA_IMG or UPDATE_TYPE == FULL_IMG:
 134                         if len(sys.argv) == 7:
 135                                 #Use path in better way
 136                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 137
 138                 global DIFF_UTIL
 139                 global DIFFPATCH_UTIL
 140                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 141                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 142                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 143                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 144                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 145                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 146                                 sys.exit(1)
 147
 148                 start = datetime.datetime.now().time()
 149                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 150                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 151
 152                 ensure_dir_exists(OUT_DIR)
 153                 if GenerateDiffAttr == "TRUE":
 154                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 155                                 print >> sys.stderr, "Attributes missing -- ABORT"
 156                                 sys.exit(1)
 157
 158
 159                 # Should check if APT is supported on other linux flavours
 160                 cache = apt.Cache()
 161                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 162                         logging.info ('Basic utils installed')
 163                 else:
 164                         print >> sys.stderr, "Basic utils missing -- ABORT"
 165                         sys.exit(1)
 166
 167                 if UPDATE_TYPE == FULL_IMG:
 168                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 169                 elif UPDATE_TYPE == DELTA_IMG:
 170                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 171                 elif UPDATE_TYPE == DELTA_FS:
 172                         AttributeFile = ATTR_NEW
 173                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 174                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 175                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 176                         New_files, New_dirs = Get_Files(BASE_NEW)
 177                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 178
 179                         if not UPDATE_CFG_PATH == EMPTY:
 180                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 181
 182
 183                 elif UPDATE_TYPE == EXTRA:
 184                         print('UPDATE_TYPE ---- EXTRA')
 185                 elif UPDATE_TYPE == PRE_UA:
 186                         print('UPDATE_TYPE ---- PRE_UA')
 187                 else:
 188                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 189
 190                 if GenerateDiffAttr == "TRUE":
 191                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 192                                 os.remove(ATTR_OLD)
 193                                 os.remove(ATTR_NEW)
 194                 end = datetime.datetime.now().time()
 195
 196                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 197                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 198                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 199                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 200
 201         except:
 202                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 203                 raise
 204
 205
 206 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 207         f = open(UPDATE_CFG_PATH, 'r')
 208         lines = f.readlines()
 209         f.close()
 210         f = open(UPDATE_CFG_PATH, 'w')
 211         for line in lines:
 212                 ConfigItems = line.split()
 213                 if ConfigItems[0] == DELTA_BIN:
 214                         DELTA = ConfigItems[1]
 215                         logging.info ('Updating %s config' % DELTA_BIN)
 216                         line = line.rstrip('\n')
 217                         Value = MEM_REQ
 218                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 219                         f.write(line)
 220                 else:
 221                         f.write(line)
 222         f.close()
 223
 224 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 225         #for sizes
 226
 227         ZIMAGE_SCRIPT = COMMON_BIN_PATH+'./dzImagescript.sh'
 228         ZIMAGE_OLD = BASE_OLD+'_unpacked'
 229         ZIMAGE_NEW = BASE_NEW+'_unpacked'
 230         DZIMAGE_HEADER = 'UnpackdzImage'
 231         DZIMAGE_SEP = ':'
 232
 233         oldsize_d= os.path.getsize(BASE_OLD)
 234         newsize_d= os.path.getsize(BASE_NEW)
 235         SHA_BIN_DEST= hash_file(BASE_NEW)
 236         SHA_BIN_BASE=hash_file(BASE_OLD)
 237
 238         #incase UPDATE CFG is empty
 239         DELTA = DELTA_BIN
 240         SS_UpdateSize(BASE_OLD, BASE_NEW)
 241         #Should throw error if PART NAME NOT found??
 242         if not UPDATE_CFG_PATH == EMPTY:
 243                 f = open(UPDATE_CFG_PATH, 'r')
 244                 lines = f.readlines()
 245                 f.close()
 246                 f = open(UPDATE_CFG_PATH, 'w')
 247                 for line in lines:
 248                         ConfigItems = line.split()
 249                         if ConfigItems[0] == DELTA_BIN:
 250                                 logging.info ('Updating %s config' % DELTA_BIN)
 251                                 DELTA = ConfigItems[1]
 252                                 line = line.rstrip('\n')
 253                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 254                                 f.write(line)
 255                         else:
 256                                 f.write(line)
 257                 f.close()
 258
 259         #Any validation checks required?
 260         if (DELTA_BIN == "zImage" or DELTA_BIN == "dzImage" or DELTA_BIN == "KERNEL" or DELTA_BIN == "BOOT") and SUPPORT_DZIMAGE == "TRUE":
 261
 262                 #Unpack Old and New Images for creating delta
 263                 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_OLD])
 264                 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_NEW])
 265
 266                 DeltaFiles = []
 267                 Old_files, Old_dirs = Get_Files(ZIMAGE_OLD)
 268                 New_files, New_dirs = Get_Files(ZIMAGE_NEW)
 269
 270                 patchLoc = '%s/%s_temp' % (OUT_DIR, DELTA_BIN)
 271                 ensure_dir_exists(patchLoc)
 272
 273                 for elt in New_files:
 274                         if elt in Old_files:
 275                                 src_file = ZIMAGE_OLD+'/'+elt
 276                                 dst_file = ZIMAGE_NEW+'/'+elt
 277                                 if not filecmp.cmp(src_file, dst_file):
 278                                         patch = '%s/%s' % (patchLoc,elt)
 279                                         DeltaFiles.append(patch)
 280                                         subprocess.call([DIFF_UTIL,src_file,dst_file,patch])
 281                                         logging.info('Make dz Image %s <--> %s ==> %s %s' % (src_file, dst_file , DELTA_BIN, patch))
 282
 283                 #Append all delta files to make image.delta
 284
 285                 #HEADER FORMAT MAGICNAME:FILECOUNT:[FILENAME:FILESIZE:][FILECONTENT/S]
 286                 HeaderStr = DZIMAGE_HEADER+DZIMAGE_SEP+'%d' % len(DeltaFiles)
 287                 HeaderStr = HeaderStr+DZIMAGE_SEP
 288
 289                 with open(OUT_DIR+'/'+DELTA, 'w') as DeltaFile:
 290                         for fname in DeltaFiles:
 291                                 DeltaSize = os.path.getsize(fname)
 292                                 HeaderStr = HeaderStr+path_leaf(fname)+DZIMAGE_SEP+'%d' % DeltaSize
 293                                 HeaderStr = HeaderStr+DZIMAGE_SEP
 294                         #Using 128 bytes as max Header.
 295                         logging.info('zImage Header - %s' % HeaderStr.ljust(128,'0'))
 296                         DeltaFile.write(HeaderStr.ljust(128,'0'))
 297                         for fname in DeltaFiles:
 298                                 with open(fname) as infile:
 299                                         DeltaFile.write(infile.read())
 300                                         infile.close()
 301
 302                 DeltaFile.close()
 303                 shutil.rmtree(patchLoc)
 304                 shutil.rmtree(ZIMAGE_OLD)
 305                 shutil.rmtree(ZIMAGE_NEW)
 306                 #Do we need to incorprate Max memory required for backup??
 307
 308         else:
 309                 patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 310                 logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 311                 subprocess.call([DIFF_UTIL,BASE_OLD,BASE_NEW,patchLoc])
 312
 313
 314
 315 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 316         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 317         oldsize_d= os.path.getsize(BASE_OLD)
 318         newsize_d= os.path.getsize(BASE_NEW)
 319         SHA_BIN_DEST= hash_file(BASE_NEW)
 320         SHA_BIN_BASE=hash_file(BASE_OLD)
 321         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 322         SS_UpdateSize(BASE_OLD, BASE_NEW)
 323
 324         if not UPDATE_CFG_PATH == EMPTY:
 325                 f = open(UPDATE_CFG_PATH, 'r')
 326                 lines = f.readlines()
 327                 f.close()
 328                 f = open(UPDATE_CFG_PATH, 'w')
 329                 for line in lines:
 330                         ConfigItems = line.split()
 331                         if ConfigItems[0] == DELTA_BIN:
 332                                 logging.info ('Updating %s config' % DELTA_BIN)
 333                                 DELTA = ConfigItems[1]
 334                                 line = line.rstrip('\n')
 335                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 336                                 f.write(line)
 337                         else:
 338                                 f.write(line)
 339                 f.close()
 340
 341 def zipdir(path, zip):
 342     for root, dirs, files in os.walk(path):
 343         for file in files:
 344             zip.write(os.path.join(root, file))
 345
 346 def ensure_dir_exists(path):
 347         if not os.path.exists(path):
 348                 os.makedirs(path)
 349                 #shutil.rmtree(path)
 350         #os.makedirs(path)
 351
 352
 353 def path_leaf(path):
 354     head, tail = ntpath.split(path) #This is for windows?? Recheck
 355     return tail
 356
 357 def path_head(path):
 358     head, tail = ntpath.split(path)
 359     return head
 360
 361 def difflines(list1, list2):
 362     c = set(list1).union(set(list2))
 363     d = set(list1).intersection(set(list2))
 364     return list(c-d)
 365
 366 #Creating Diff between OLD and NEW attribute files v12
 367 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 368         if GenerateDiffAttr == "FALSE":
 369                 return
 370         with open(ATTR_OLD, 'r') as f_old:
 371                 lines1 = set(f_old.read().splitlines())
 372
 373         with open(ATTR_NEW, 'r') as f_new:
 374                 lines2 = set(f_new.read().splitlines())
 375
 376         lines = difflines(lines2, lines1)
 377         with open(ATTR_FILE, 'w+') as file_out:
 378                 for line in lines:
 379                         if line not in lines1:
 380                                 logging.info('Diff_AttrFiles - %s' % line)
 381                                 file_out.write(line+'\n')
 382
 383         f_new.close()
 384         f_old.close()
 385         file_out.close()
 386
 387
 388
 389 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 390         #Full File Path should MATCH
 391         if GenerateDiffAttr == "FALSE":
 392                 return
 393         FilePath = '"/'+RequestedPath+'"'
 394         #print ('FilePath - %s'% (FilePath))
 395         with open(AttributeFile) as f:
 396                 for line in f:
 397                         if FilePath in line:
 398                                 if Type == SYMLINK_TYPE:
 399                                         Sym_Attibutes.append(line)
 400                                 else:
 401                                         File_Attibutes.append(line)
 402
 403
 404 '''This function returns the SHA-1 hash of the file passed into it'''
 405 def hash_file(filename):
 406
 407    # make a hash object
 408    h = hashlib.sha1()
 409
 410    # open file for reading in binary mode
 411    with open(filename,'rb') as file:
 412        # loop till the end of the file
 413        chunk = 0
 414        while chunk != b'':
 415            # read only 1024 bytes at a time
 416            chunk = file.read(1024*1024)
 417            h.update(chunk)
 418
 419    # return the hex representation of digest
 420    return h.hexdigest()
 421
 422 def find_dupes_dir(BASE_OLD, BASE_NEW):
 423         dups = {}
 424         fdupes = {}
 425         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 426         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 427         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 428                 #print('Scanning %s...' % rootbase)
 429                 for filename in fileListB:
 430                         path = os.path.join(rootbase, filename)
 431                         if os.path.islink(path):
 432                                 continue
 433                         # Calculate hash
 434                         file_hash = hash_file(path)
 435                         dups[file_hash] = path
 436
 437         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 438                 #print('Scanning %s...' % roottarget)
 439                 for filename in fileListT:
 440                         # Get the path to the file
 441                         path = os.path.join(roottarget, filename)
 442                         if os.path.islink(path):
 443                                 continue
 444                         # Calculate hash
 445                         file_hash = hash_file(path)
 446                         # Add or append the file path
 447                         if file_hash in dups:
 448                                 BaseStr = dups.get(file_hash)
 449                                 Baseloc = path.find('/')
 450                                 TarLoc = BaseStr.find('/')
 451                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 452                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 453                                         fdupes[path] = BaseStr
 454         logging.info('Total Duplicate files %d' % (len(fdupes)))
 455         return fdupes
 456
 457
 458 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 459         dups = {}
 460         fdupes = {}
 461         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 462
 463         for filename in fileListB:
 464                 Src_File = BASE_OLD+'/'+filename
 465                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 466                         continue
 467                 # Calculate hash
 468                 file_hash = hash_file(Src_File)
 469                 dups[file_hash] = Src_File
 470
 471
 472         for filename in fileListT:
 473                 Dest_File = BASE_NEW+'/'+filename
 474                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 475                         continue
 476                 # Calculate hash
 477                 file_hash = hash_file(Dest_File)
 478                 if file_hash in dups:
 479                         BaseStr = dups.get(file_hash)
 480                         Baseloc = BaseStr.find('/')
 481                         if not BaseStr[Baseloc:] == filename:
 482                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 483                                 fdupes[BaseStr] = filename
 484
 485         logging.info('Total Duplicate files %d' % (len(fdupes)))
 486         return fdupes
 487
 488 def SS_UpdateSize(src_file, dst_file):
 489         global MEM_REQ
 490         global MEM_FILE
 491         oldsize_d= os.path.getsize(src_file)
 492         newsize_d= os.path.getsize(dst_file)
 493         if oldsize_d >= newsize_d:
 494                 Max = newsize_d
 495         else:
 496                 Max = oldsize_d
 497         if MEM_REQ < Max:
 498                 MEM_REQ = Max
 499                 MEM_FILE = dst_file
 500
 501
 502
 503 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 504         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 505         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 506
 507         # First let's fill up these categories
 508         files_new = []
 509         files_removed = []
 510         Dir_removed = []
 511         Dir_Added = []
 512         files_changed = []
 513         files_unchanged = []
 514         files_renamed = []
 515         File_Attibutes = []
 516         Sym_Attibutes = []
 517
 518         files_Del_List = {}
 519         files_New_List = {}
 520         MyDict_Patches = {}
 521
 522
 523
 524         PWD = os.getcwd()
 525
 526         # Generate NEW List
 527         for elt in New_files:
 528                 if elt not in Old_files:
 529                         files_new.append(elt)
 530                         logging.info('New files %s' % elt)
 531
 532         # Generate Delete List
 533         for elt in Old_files:
 534                 if elt not in New_files:
 535                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 536                         files_removed.append(elt)
 537                         logging.info('Old files %s' % elt)
 538
 539
 540         for elt in Old_dirs:
 541                 #print('List of Old Dirs %s' % elt)
 542                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 543                 if elt not in New_dirs:
 544                         Dir_removed.append(elt)
 545                         logging.info('Old Dirs %s' % elt+'/')
 546
 547         for elt in New_dirs:
 548                 if elt not in Old_dirs:
 549                         Dir_Added.append(elt)
 550                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 551
 552         # What files have changed contents but not name/path?
 553         for elt in New_files:
 554                 if elt in Old_files:
 555                         #Both are symbolic linkes and they differ
 556                         src_file = BASE_OLD+'/'+elt
 557                         dst_file = BASE_NEW+'/'+elt
 558                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 559                         if os.path.islink(src_file) and os.path.islink(dst_file):
 560                                 if not os.readlink(src_file) == os.readlink(dst_file):
 561                                         files_changed.append(elt)
 562                                         #print('%d Sym link files changed' % len(files_changed))
 563                                         logging.info('Sym links Changed - %s' % elt)
 564                                 else:
 565                                         files_unchanged.append(elt)
 566                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 567                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 568                                 if not filecmp.cmp(src_file, dst_file):
 569                                         files_changed.append(elt)
 570                                         #print('%d Normal files changed' % len(files_changed))
 571                                         #print('Files Changed - %s' % elt)
 572                                 else:
 573                                         files_unchanged.append(elt)
 574                         #File types differ between BASE and TARGET
 575                         else:
 576                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 577                                 #Both file types have changed and they differ
 578                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 579                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 580                                 files_removed.append(elt)
 581                                 files_new.append(elt)
 582
 583
 584         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 585         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 586
 587         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 588         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 589                 with open(VERBATIM_LIST, 'r') as F_News:
 590                         lines = set(F_News.read().splitlines())
 591                 for line in lines:
 592                         if line in files_changed:
 593                                 files_changed.remove(line)
 594                                 files_removed.append(line)
 595                         if line in files_new:
 596                                 files_new.remove(line)
 597
 598         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 599         if SUPPORT_RENAME == "TRUE":
 600                 for elt in files_removed:
 601                         if os.path.isfile(BASE_OLD+'/'+elt):
 602                                 FileName = path_leaf(elt)
 603                                 entries = re.split('[0-9]' , FileName)
 604                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 605                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 606                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 607                                 if len(entries[0]) > 0:
 608                                         files_Del_List.update({entries[0]: elt})
 609
 610                 for elt in files_new:
 611                         if os.path.isfile(BASE_NEW+'/'+elt):
 612                                 FileName = path_leaf(elt)
 613                                 entries = re.split('[0-9]' , FileName)
 614                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 615                                 if len(entries[0]) > 0:
 616                                         files_New_List.update({entries[0]: elt})
 617
 618                 for key, value in files_Del_List.iteritems():
 619                         #print('Key value pair -%s -%s' % (key, value))
 620                         if key in files_New_List:
 621                                 # this file is the same name in both!
 622                                 src_file = BASE_OLD+'/'+value
 623                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 624                                 olddirpath = path_head(files_New_List[key])
 625                                 newdirpath = path_head(value)
 626                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 627                                         logging.debug('Cannot diff as one of them is Symlink')
 628                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 629                                         logging.debug('Cannot diff as one of them is dir')
 630                                 else:
 631                                         #Pick the best diff of same type and diff names
 632                                         files_renamed.append([files_New_List[key], value])
 633                                         files_removed.remove(value)
 634                                         files_new.remove(files_New_List[key])
 635
 636         '''
 637         Patch Section
 638                 Partition.txt contains Protocol for UPI
 639                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 640         '''
 641         Sym_Diff_Cnt = 0
 642         Sym_New_Cnt = 0;
 643         Del_Cnt = 0
 644         New_Cnt = 0
 645         Diff_Cnt = 0
 646         Move_Cnt = 0
 647         Verbatim_Cnt = 0
 648         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 649         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 650         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 651
 652         print("writing diff'ed changed files...")
 653         for elt in files_changed:
 654                 dst_file = BASE_NEW+'/'+elt
 655                 src_file = BASE_OLD+'/'+elt
 656                 #Both files are symbolic links and they differ
 657                 if os.path.islink(dst_file) and os.path.islink(src_file):
 658                 #Both are symlinks and they differ
 659                         logging.debug(' File Changed is Link %s ' % dst_file)
 660                         patch = os.readlink(dst_file)
 661                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 662                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 663                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 664                 #Both are NORMAL files and they differ
 665                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 666                         #Both are files and they differ
 667                         Diff_Cnt = Diff_Cnt + 1
 668                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 669                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 670                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 671                         SS_UpdateSize(src_file, dst_file)
 672
 673                         FORMAT = "REG"
 674                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 675                         if ret is not 0:
 676                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 677                                 files_new.append(elt)
 678                                 Diff_Cnt = Diff_Cnt - 1
 679                         else:
 680                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 681
 682                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 683                 #Both differ but they are of diff types
 684                 else:
 685                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 686                         files_removed.append(elt)
 687                         files_new.append(elt)
 688
 689         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 690         for oldpath, newpath in fdupes.iteritems():
 691                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 692
 693         for elt in files_removed:
 694                 src_file = BASE_OLD+'/'+elt
 695                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 696                 if src_file in fdupes.keys():
 697                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 698                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 699                         Move_Cnt = Move_Cnt + 1
 700                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 701                         files_removed.remove(elt)
 702                         files_new.remove(fdupes[src_file])
 703
 704         #Should be placed after removing duplicates, else they will be filtered here.
 705         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 706         #       First Step: Sort & Filter out unwanted files
 707         # Minimum condition used is,
 708         #       1. File name should match 70%
 709         #       2. Extensions should be same
 710         #       3. File name length shd b greater than 3 char
 711         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 712         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 713
 714         Base_DelList = files_removed[:]
 715         Base_NewList = files_new[:]
 716         DelList = sorted(Base_DelList, key=path_leaf)
 717         NewList = sorted(Base_NewList, key=path_leaf)
 718         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 719
 720         Filter1 = []
 721         Filter2 = []
 722         #Remove unwanted items which we cant make diff with for rename logic
 723         for file in DelList:
 724                 if os.path.islink(BASE_OLD+'/'+file):
 725                         continue
 726                 elif os.path.isdir(BASE_OLD+'/'+file):
 727                         continue
 728                 else:
 729                         Filter1.append(file)
 730                         #logging.debug('Sorted del list - %s' % (file))
 731
 732         DelList = Filter1
 733
 734         for file in NewList:
 735                 if os.path.islink(BASE_NEW+'/'+file):
 736                         continue
 737                 elif os.path.isdir(BASE_NEW+'/'+file):
 738                         continue
 739                 elif len(path_leaf(file)) <= 3:
 740                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 741                         continue
 742                 else:
 743                         Filter2.append(file)
 744
 745         NewList = Filter2
 746
 747         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 748
 749         for new_file in NewList:
 750                 R_Flag = 'FALSE';
 751                 DirPathNew = path_head(new_file)
 752                 FileNameNew = path_leaf(new_file)
 753                 DiffSize = 0
 754                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 755                 New_fs = winning_patch_sz
 756                 winning_file = ''
 757
 758                 for del_file in DelList:
 759                         FileNameOld = path_leaf(del_file)
 760                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 761                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 762                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 763                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 764                                 v1 = abs(New_fs-Del_fs)
 765                                 v2 = (New_fs+Del_fs)/2
 766                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 767                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 768                                         continue
 769                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 770                                 R_Flag = 'TRUE';
 771                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 772                                 if (DiffSize < 0.8 * winning_patch_sz):
 773                                         winning_patch_sz = DiffSize
 774                                         winning_file = del_file
 775                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 776                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 777                                 break;
 778                 if len(winning_file) > 0:
 779                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 780                         files_renamed.append([new_file, winning_file])
 781                         DelList.remove(winning_file)
 782                         files_removed.remove(winning_file)
 783                         files_new.remove(new_file)
 784
 785         #********************** Files should NOT be deleted for any such renames ***********************
 786
 787         if SUPPORT_RENAME == "TRUE":
 788                 for elt in files_renamed:
 789                         src_file = BASE_OLD+'/'+elt[1]
 790                         dst_file = BASE_NEW+'/'+elt[0]
 791                         Diff_Cnt = Diff_Cnt + 1
 792                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 793                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 794                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 795                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 796                         # Should be careful of renaming files??
 797                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 798                         # What if folder path has numerics??
 799
 800                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 801                                 #This case never occurs??
 802                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 803                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 804                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 805                                 if filecmp.cmp(src_file, dst_file):
 806                                         Move_Cnt = Move_Cnt + 1
 807                                         Diff_Cnt = Diff_Cnt - 1
 808                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 809                                 else:
 810                                         FORMAT = "REG"
 811                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 812                                         if ret is not 0:
 813                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 814                                                 files_new.append(elt)
 815                                                 Diff_Cnt = Diff_Cnt - 1
 816                                         else:
 817                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 818
 819                                 SS_UpdateSize(src_file, dst_file)
 820                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 821
 822
 823         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 824         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 825
 826         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 827                 with open(VERBATIM_LIST, 'r') as F_News:
 828                         lines = set(F_News.read().splitlines())
 829                 for line in lines:
 830                         if not line in files_new:
 831                                 if os.path.exists(BASE_NEW+'/'+line):
 832                                         files_new.append(line)
 833                                         Verbatim_Cnt = Verbatim_Cnt+1
 834                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 835
 836
 837
 838         for elt in files_removed:
 839                 #if files are part of patches after renaming, we shd remove them as part of removed.
 840                 src_file = BASE_OLD+'/'+elt
 841                 if os.path.islink(src_file):
 842                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 843                 elif os.path.isdir(src_file):
 844                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 845                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 846                 else:
 847                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 848                 logging.debug(' File Deleted %s' % src_file)
 849                 Del_Cnt = Del_Cnt + 1
 850
 851         Dir_removed.sort(reverse=True)
 852         for elt in Dir_removed:
 853                 #if Dir is empty, add it to the removed list.
 854                 src_file = BASE_OLD+'/'+elt
 855                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 856                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 857                 if os.path.isdir(src_file):
 858                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 859                         Del_Cnt = Del_Cnt + 1
 860                         logging.debug(' Dir Deleted- %s' % src_file)
 861
 862
 863         for elt in files_new:
 864                 dst_file = BASE_NEW+'/'+elt
 865                 newfiles_dest_path = 'system/'
 866                 ensure_dir_exists(newfiles_dest_path)
 867                 if os.path.islink(dst_file):
 868                         patch = os.readlink(dst_file)
 869                         logging.debug(' File New Links %s' % elt)
 870                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 871                         #What if this is only a new sym link and folder already exists??? Should recheck
 872                         destpath = newfiles_dest_path + elt
 873                         if not os.path.exists(path_head(destpath)):
 874                                 os.makedirs(path_head(destpath))
 875                                 logging.info('New SymLink - Adding missing Dir')
 876                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 877                         Sym_New_Cnt = Sym_New_Cnt + 1
 878                 elif os.path.isdir(dst_file): # We create just empty directory here
 879                         destpath = newfiles_dest_path + elt
 880                         if not os.path.exists(destpath):
 881                                 os.makedirs(destpath)
 882                                 logging.debug(' File New Dir %s' % destpath)
 883                                 New_Cnt = New_Cnt + 1
 884                 else:
 885                         New_Cnt = New_Cnt + 1
 886                         #newfiles_dest_path = OUT_DIR + '/system/'
 887                         destpath = newfiles_dest_path + elt
 888                         destdir = os.path.dirname(destpath)
 889                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 890
 891                         if not os.path.isdir(destdir):
 892                                 try:
 893                                         os.makedirs(destdir)
 894                                 except:
 895                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 896                                         raise
 897
 898                         try:
 899                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 900                                         shutil.copy2(dst_file, destpath)
 901                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 902                         except:
 903                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 904                                 raise
 905
 906         for elt in Dir_Added:
 907                 newfiles_dest_path = 'system/'
 908                 ensure_dir_exists(newfiles_dest_path)
 909                 destpath = newfiles_dest_path + elt
 910                 if not os.path.exists(destpath):
 911                         os.makedirs(destpath)
 912                         logging.debug(' DirList New Dir %s' % destpath)
 913                         New_Cnt = New_Cnt + 1
 914
 915         #Base directory should be system
 916         print 'Compressing New files'
 917         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 918                 WorkingDir = os.getcwd()
 919                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 920                 logging.info('Curr Working Dir - %s' % os.getcwd())
 921                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 922                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 923                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 924                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 925                 os.chdir(WorkingDir)
 926                 shutil.rmtree(NEW_FILES_PATH)
 927                 # use 7z a system.7z ./*
 928
 929         #logging.info('%d Dir to be removed' % len(Dir_removed))
 930         logging.info('%d files unchanged' % len(files_unchanged))
 931         logging.info('%d files files_renamed' % len(files_renamed))
 932         logging.info('%d files NEW' % len(files_new))
 933         logging.info('%d File attr' % len(File_Attibutes))
 934         logging.info('%d Sym attr' % len(Sym_Attibutes))
 935         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 936         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 937
 938         #There could be duplicates, TODO, can check before adding..
 939         ATTR_FILE_D = open(ATTR_FILE,'a+')
 940         for elt in File_Attibutes:
 941                 ATTR_FILE_D.write(elt)
 942         for elt in Sym_Attibutes:
 943                 ATTR_FILE_D.write(elt)
 944
 945         ATTR_FILE_D.close()
 946
 947         Partition_Doc_SymLinks.close()
 948         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 949         Partition_Doc.write(Partition_Read_SymLinks.read())
 950         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 951         Partition_Doc_SymLinks.close()
 952         Partition_Doc.close()
 953         os.remove(SymLinkDoc)
 954
 955         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 956                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 957                 logging.info('No Delta Generated for %s' % PART_NAME)
 958                 shutil.rmtree(OUT_DIR)
 959
 960
 961 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 962
 963         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 964         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 965         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 966         temp_apk = '../'+g_output_dir+'/'+b_apk
 967         Patch = 'Patch_'+b_apk
 968         ensure_dir_exists(Patch)
 969         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 970
 971         #Size issue on Device side?? shd check this
 972         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 973         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 974                 lines = set(f_new.read().splitlines())
 975                 for line in lines:
 976                         #print('Action ==> %s' % line)
 977                         #Action, Path, Patch = line.split('|')
 978                         Items = line.split('|')
 979                         Action = Items[0]
 980                         Path = Items[1]
 981                         ActualPath = a_folder+'/'+Path
 982                         PatchPath = Patch+'/'+Path
 983                         SrcPath = g_output_dir+'/'+path_leaf(Path)
 984                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 985                         if line[0] == 'c':
 986                                 patchName = g_output_dir+'/'+Items[2]
 987                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 988                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
 989                                 WorkingDir = os.getcwd()
 990                                 os.chdir(WorkingDir+"/"+"temp_a")
 991                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 992                                 os.chdir(WorkingDir)
 993                         elif line[0] == 's':
 994                                 WorkingDir = os.getcwd()
 995                                 os.chdir(WorkingDir+"/"+"temp_a")
 996                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
 997                                 os.chdir(WorkingDir)
 998                         else:
 999                                 print('Apply_Container_Delta - Unknown Error')
1000         #print('Touch all files and set common attributes for DIFF generation')
1001         WorkingDir = os.getcwd()
1002         os.chdir(WorkingDir+"/"+Patch)
1003
1004         CONTAINER_DATE = '200011111111.11'
1005         CONTAINER_MODE = '0755'
1006         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
1007         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
1008         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
1009
1010         print 'Update Intermediate Archive'
1011         #subprocess.call(['zip','-ryX', b_apk, '*'])
1012         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
1013         os.chdir(WorkingDir)
1014         #print('Apply Path completed - Now create diff for this and place in patch folder')
1015         #print os.getcwd()
1016         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
1017         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
1018         patchLoc = '%s/%s' % (g_output_dir, patchName)
1019
1020         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
1021
1022         #Only on HOST... for testing
1023         if TEST_MODE == 'TRUE':
1024                 UpgradedName = '%s_Upgraded' % (b_apk)
1025                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
1026
1027         #This is file only with NEWS and empty diffs and same files.
1028         if TEST_MODE == 'FALSE':
1029                 os.remove(g_output_dir+'/'+b_apk)
1030                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
1031                 shutil.rmtree(Patch)
1032
1033 def IsSymlink(info):
1034   return (info.external_attr >> 16) == 0120777
1035
1036 def NewFiles(src, dest):
1037         print src,dest
1038         subprocess.call(['cp','-rp', src,dest])
1039     #try:
1040                 #shutil.copytree(src, dest)
1041     #except OSError as e:
1042         # If the error was caused because the source wasn't a directory
1043         #if e.errno == errno.ENOTDIR:
1044             #shutil.copy2(src, dest)
1045         #else:
1046             #print('Directory not copied. Error: %s' % e)
1047
1048 def measure_two_filediffs(src, dst):
1049         patchLoc = 'temp.patch'
1050         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
1051         result_size = os.path.getsize(patchLoc)
1052         os.remove(patchLoc)
1053         return result_size
1054
1055 def Get_Files(path):
1056         all_files = []
1057         all_dirs = []
1058
1059         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1060                 for directory in directories:
1061                         #DirName = os.path.join(root+'/',directory)
1062                         DirName = os.path.join(root,directory)
1063                         if os.path.islink(DirName):
1064                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1065                                 all_files.append(os.path.relpath(DirName, path))
1066                         elif not os.listdir(DirName):
1067                                 #print('*****Empty Directory******* -%s', DirName)
1068                                 #This should NOT be appended ??? Empty dir shd b considered
1069                                 all_dirs.append(os.path.relpath(DirName, path))
1070                         else:
1071                                 all_dirs.append(os.path.relpath(DirName, path))
1072                 for filename in filenames:
1073                         FileName = os.path.join(root,filename)
1074                         all_files.append(os.path.relpath(FileName, path))
1075
1076         all_files.sort()
1077         all_dirs.sort()
1078         return all_files, all_dirs
1079
1080
1081 USAGE_DOCSTRING = """
1082       Generate Delta using BASEOLD AND BASE NEW
1083           Attributes is optional
1084           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1085 """
1086
1087 def Usage(docstring):
1088   print docstring.rstrip("\n")
1089   print COMMON_DOCSTRING
1090
1091
1092
1093 if __name__ == '__main__':
1094         main()
1095