mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4
   5
   6 if sys.hexversion < 0x02040000:
   7   print >> sys.stderr, "Python 2.4 or newer is required."
   8   sys.exit(1)
   9
  10 import sys
  11 import os
  12 import filecmp
  13 import shutil
  14 import subprocess
  15 import re
  16 import ntpath
  17 import zipfile
  18 import datetime
  19 import hashlib
  20 import operator
  21 import locale
  22 import errno
  23 import logging
  24 import glob
  25 import apt
  26 import stat
  27 '''
  28 Diff two folders and create delta using SS_BSDIFF
  29 Will maintain same format of script that will be generated when we use diffutil
  30
  31 1. Create a list of files in each Base folders,
  32 2. These files will fall into one these below categories:
  33         1) Only in OLD - Should be deleted
  34         2) Only in NEW - Should be added or renamed accordingly
  35         3) File exists in both directories but contents are different - Create Diff.
  36         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  37         5) Duplicates in the list of Deletes and News
  38         6) Close matching diffs even though name changes across directories. (for matching extension)
  39         7) Clearing empty directories after Moves or diffs under Rename.
  40         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  41
  42 Current Case
  43 1. Given two folders, from list of REMOVED and NEW files find if there
  44 is version change and create diff between them
  45
  46 TODO
  47 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  48 Catching errors at all stages. SHOULD exit & return error in case of failure
  49 '''
  50
  51 def global_paths():
  52         global DIFF_UTIL
  53         global ZIPUTIL
  54         global NEW_FILES_PATH
  55         global NEW_FILES_FOLDER
  56         global NEW_FILES_ZIP_NAME
  57         global SYMLINK_TYPE
  58         global ATTR_DOC_EXT
  59         global SYMLINK_DOC_NAME
  60         global DIFF_PREFIX
  61         global DIFF_SUFFIX
  62         global SUPPORT_RENAME
  63         global NEW_PREFIX
  64         global DIFFPATCH_UTIL
  65         global SUPPORT_CONTAINERS
  66         global FULL_IMG
  67         global DELTA_IMG
  68         global DELTA_FS
  69         global EXTRA
  70         global COMMON_BIN_PATH
  71         global MEM_REQ
  72         global EMPTY
  73         global VERBATIM_LIST
  74         global MEM_FILE
  75
  76 COMMON_BIN_PATH = "../../common/bin/"
  77 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  78 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  79 ZIPUTIL = "p7zip "
  80 #ZIPUTIL = "7z a system.7z "
  81 NEW_FILES_PATH = "system"
  82 NEW_FILES_FOLDER  = "system"
  83 NEW_FILES_ZIP_NAME = "system.7z"
  84 SYMLINK_TYPE = "SYM"
  85 ATTR_DOC_EXT = "_attr.txt"
  86 SYMLINK_DOC_NAME = "_sym.txt"
  87 PART_DOC_EXT = ".txt"
  88 DIFF_PREFIX = "diff"
  89 DIFF_SUFFIX = ".delta"
  90 NEW_PREFIX = 'new'
  91 FULL_IMG = "FULL_IMG"
  92 DELTA_IMG = "DELTA_IMG"
  93 DELTA_FS = "DELTA_FS"
  94 EXTRA = "EXTRA"
  95 LOGFILE = "Delta.log"
  96 VERBATIM_LIST = "Verbatim_List.txt"
  97 EMPTY = ""
  98 MEM_REQ = 0
  99 MEM_FILE = "NULL"
 100
 101 SUPPORT_RENAME = "TRUE" #Use appropriate name
 102 SUPPORT_CONTAINERS = "FALSE"
 103 SUPPORT_DZIMAGE = "TRUE"
 104 SUPPORT_VERBATIM = "TRUE"
 105
 106 TEST_MODE = "FALSE"
 107
 108 def main():
 109         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 110         global AttributeFile
 111         global GenerateDiffAttr
 112         try:
 113
 114                 if len(sys.argv) < 5:
 115                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 116                 UPDATE_TYPE = sys.argv[1]
 117                 PART_NAME = sys.argv[2]  # lets make this also optional
 118
 119                 BASE_OLD = sys.argv[3]
 120                 BASE_NEW = sys.argv[4]
 121                 OUT_DIR = sys.argv[5]
 122                 ATTR_OLD = EMPTY
 123                 ATTR_NEW = EMPTY
 124                 UPDATE_CFG_PATH = EMPTY
 125                 GenerateDiffAttr = "FALSE"
 126                 if UPDATE_TYPE == DELTA_FS:
 127                         #instead of arguments check it in outdirectory ?
 128                         if len(sys.argv) == 9:
 129                                 ATTR_OLD = sys.argv[6]
 130                                 ATTR_NEW = sys.argv[7]
 131                                 UPDATE_CFG_PATH = '../'+sys.argv[8]
 132                                 GenerateDiffAttr = "TRUE"
 133
 134                 elif UPDATE_TYPE == DELTA_IMG or UPDATE_TYPE == FULL_IMG:
 135                         if len(sys.argv) == 7:
 136                                 #Use path in better way
 137                                 UPDATE_CFG_PATH = '../'+sys.argv[6]
 138
 139                 global DIFF_UTIL
 140                 global DIFFPATCH_UTIL
 141                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 142                         DIFF_UTIL = COMMON_BIN_PATH+DIFF_UTIL
 143                         DIFFPATCH_UTIL = COMMON_BIN_PATH+DIFFPATCH_UTIL
 144                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 145                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 146                                 logging.info ('Diff Util Does NOT exist -- ABORT')
 147                                 sys.exit(1)
 148
 149                 start = datetime.datetime.now().time()
 150                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 151                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]'% (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 152
 153                 ensure_dir_exists(OUT_DIR)
 154                 if GenerateDiffAttr == "TRUE":
 155                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 156                                 print >> sys.stderr, "Attributes missing -- ABORT"
 157                                 sys.exit(1)
 158
 159
 160                 # Should check if APT is supported on other linux flavours
 161                 cache = apt.Cache()
 162                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 163                         logging.info ('Basic utils installed')
 164                 else:
 165                         print >> sys.stderr, "Basic utils missing -- ABORT"
 166                         sys.exit(1)
 167
 168                 if UPDATE_TYPE == FULL_IMG:
 169                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 170                 elif UPDATE_TYPE == DELTA_IMG:
 171                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 172                 elif UPDATE_TYPE == DELTA_FS:
 173                         AttributeFile = ATTR_NEW
 174                         ATTR_FILE = OUT_DIR+'/'+PART_NAME+ATTR_DOC_EXT
 175                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 176                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 177                         New_files, New_dirs = Get_Files(BASE_NEW)
 178                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 179
 180                         if not UPDATE_CFG_PATH == EMPTY:
 181                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 182
 183
 184                 elif UPDATE_TYPE == EXTRA:
 185                         print('UPDATE_TYPE ---- EXTRA')
 186                 else:
 187                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 188
 189                 if GenerateDiffAttr == "TRUE":
 190                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 191                                 os.remove(ATTR_OLD)
 192                                 os.remove(ATTR_NEW)
 193                 end = datetime.datetime.now().time()
 194
 195                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 196                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 197                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 198                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 199
 200         except:
 201                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 202                 raise
 203
 204
 205 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 206         f = open(UPDATE_CFG_PATH, 'r')
 207         lines = f.readlines()
 208         f.close()
 209         f = open(UPDATE_CFG_PATH, 'w')
 210         for line in lines:
 211                 ConfigItems = line.split()
 212                 if ConfigItems[0] == DELTA_BIN:
 213                         DELTA = ConfigItems[1]
 214                         logging.info ('Updating %s config' % DELTA_BIN)
 215                         line = line.rstrip('\n')
 216                         Value = MEM_REQ
 217                         line = line.replace(line, line+'\t'+str(Value)+'\n')
 218                         f.write(line)
 219                 else:
 220                         f.write(line)
 221         f.close()
 222
 223 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 224         #for sizes
 225
 226         ZIMAGE_SCRIPT = COMMON_BIN_PATH+'./dzImagescript.sh'
 227         ZIMAGE_OLD = BASE_OLD+'_unpacked'
 228         ZIMAGE_NEW = BASE_NEW+'_unpacked'
 229         DZIMAGE_HEADER = 'UnpackdzImage'
 230         DZIMAGE_SEP = ':'
 231
 232         oldsize_d= os.path.getsize(BASE_OLD)
 233         newsize_d= os.path.getsize(BASE_NEW)
 234         SHA_BIN_DEST= hash_file(BASE_NEW)
 235         SHA_BIN_BASE=hash_file(BASE_OLD)
 236
 237         #incase UPDATE CFG is empty
 238         DELTA = DELTA_BIN
 239         SS_UpdateSize(BASE_OLD, BASE_NEW)
 240         #Should throw error if PART NAME NOT found??
 241         if not UPDATE_CFG_PATH == EMPTY:
 242                 f = open(UPDATE_CFG_PATH, 'r')
 243                 lines = f.readlines()
 244                 f.close()
 245                 f = open(UPDATE_CFG_PATH, 'w')
 246                 for line in lines:
 247                         ConfigItems = line.split()
 248                         if ConfigItems[0] == DELTA_BIN:
 249                                 logging.info ('Updating %s config' % DELTA_BIN)
 250                                 DELTA = ConfigItems[1]
 251                                 line = line.rstrip('\n')
 252                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 253                                 f.write(line)
 254                         else:
 255                                 f.write(line)
 256                 f.close()
 257
 258         #Any validation checks required?
 259         if (DELTA_BIN == "zImage" or DELTA_BIN == "dzImage" or DELTA_BIN == "KERNEL" or DELTA_BIN == "BOOT") and SUPPORT_DZIMAGE == "TRUE":
 260
 261                 #Unpack Old and New Images for creating delta
 262                 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_OLD])
 263                 subprocess.call([ZIMAGE_SCRIPT, '-u', BASE_NEW])
 264
 265                 DeltaFiles = []
 266                 Old_files, Old_dirs = Get_Files(ZIMAGE_OLD)
 267                 New_files, New_dirs = Get_Files(ZIMAGE_NEW)
 268
 269                 patchLoc = '%s/%s_temp' % (OUT_DIR, DELTA_BIN)
 270                 ensure_dir_exists(patchLoc)
 271
 272                 for elt in New_files:
 273                         if elt in Old_files:
 274                                 src_file = ZIMAGE_OLD+'/'+elt
 275                                 dst_file = ZIMAGE_NEW+'/'+elt
 276                                 if not filecmp.cmp(src_file, dst_file):
 277                                         patch = '%s/%s' % (patchLoc,elt)
 278                                         DeltaFiles.append(patch)
 279                                         subprocess.call([DIFF_UTIL,src_file,dst_file,patch])
 280                                         logging.info('Make dz Image %s <--> %s ==> %s %s' % (src_file, dst_file , DELTA_BIN, patch))
 281
 282                 #Append all delta files to make image.delta
 283
 284                 #HEADER FORMAT MAGICNAME:FILECOUNT:[FILENAME:FILESIZE:][FILECONTENT/S]
 285                 HeaderStr = DZIMAGE_HEADER+DZIMAGE_SEP+'%d' % len(DeltaFiles)
 286                 HeaderStr = HeaderStr+DZIMAGE_SEP
 287
 288                 with open(OUT_DIR+'/'+DELTA, 'w') as DeltaFile:
 289                         for fname in DeltaFiles:
 290                                 DeltaSize = os.path.getsize(fname)
 291                                 HeaderStr = HeaderStr+path_leaf(fname)+DZIMAGE_SEP+'%d' % DeltaSize
 292                                 HeaderStr = HeaderStr+DZIMAGE_SEP
 293                         #Using 128 bytes as max Header.
 294                         logging.info('zImage Header - %s' % HeaderStr.ljust(128,'0'))
 295                         DeltaFile.write(HeaderStr.ljust(128,'0'))
 296                         for fname in DeltaFiles:
 297                                 with open(fname) as infile:
 298                                         DeltaFile.write(infile.read())
 299                                         infile.close()
 300
 301                 DeltaFile.close()
 302                 shutil.rmtree(patchLoc)
 303                 shutil.rmtree(ZIMAGE_OLD)
 304                 shutil.rmtree(ZIMAGE_NEW)
 305                 #Do we need to incorprate Max memory required for backup??
 306
 307         else:
 308                 patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 309                 logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW , DELTA_BIN, patchLoc))
 310                 subprocess.call([DIFF_UTIL,BASE_OLD,BASE_NEW,patchLoc])
 311
 312
 313
 314 def     SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN ,UPDATE_CFG_PATH):
 315         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW ,DELTA_BIN))
 316         oldsize_d= os.path.getsize(BASE_OLD)
 317         newsize_d= os.path.getsize(BASE_NEW)
 318         SHA_BIN_DEST= hash_file(BASE_NEW)
 319         SHA_BIN_BASE=hash_file(BASE_OLD)
 320         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 321         SS_UpdateSize(BASE_OLD, BASE_NEW)
 322
 323         if not UPDATE_CFG_PATH == EMPTY:
 324                 f = open(UPDATE_CFG_PATH, 'r')
 325                 lines = f.readlines()
 326                 f.close()
 327                 f = open(UPDATE_CFG_PATH, 'w')
 328                 for line in lines:
 329                         ConfigItems = line.split()
 330                         if ConfigItems[0] == DELTA_BIN:
 331                                 logging.info ('Updating %s config' % DELTA_BIN)
 332                                 DELTA = ConfigItems[1]
 333                                 line = line.rstrip('\n')
 334                                 line = line.replace(line, line+'\t'+str(oldsize_d)+'\t\t'+str(newsize_d)+'\t\t'+str(SHA_BIN_BASE)+'\t\t'+str(SHA_BIN_DEST)+'\n')
 335                                 f.write(line)
 336                         else:
 337                                 f.write(line)
 338                 f.close()
 339
 340 def zipdir(path, zip):
 341     for root, dirs, files in os.walk(path):
 342         for file in files:
 343             zip.write(os.path.join(root, file))
 344
 345 def ensure_dir_exists(path):
 346         if not os.path.exists(path):
 347                 os.makedirs(path)
 348                 #shutil.rmtree(path)
 349         #os.makedirs(path)
 350
 351
 352 def path_leaf(path):
 353     head, tail = ntpath.split(path) #This is for windows?? Recheck
 354     return tail
 355
 356 def path_head(path):
 357     head, tail = ntpath.split(path)
 358     return head
 359
 360 def difflines(list1, list2):
 361     c = set(list1).union(set(list2))
 362     d = set(list1).intersection(set(list2))
 363     return list(c-d)
 364
 365 #Creating Diff between OLD and NEW attribute files v12
 366 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 367         if GenerateDiffAttr == "FALSE":
 368                 return
 369         with open(ATTR_OLD, 'r') as f_old:
 370                 lines1 = set(f_old.read().splitlines())
 371
 372         with open(ATTR_NEW, 'r') as f_new:
 373                 lines2 = set(f_new.read().splitlines())
 374
 375         lines = difflines(lines2, lines1)
 376         with open(ATTR_FILE, 'w+') as file_out:
 377                 for line in lines:
 378                         if line not in lines1:
 379                                 logging.info('Diff_AttrFiles - %s' % line)
 380                                 file_out.write(line+'\n')
 381
 382         f_new.close()
 383         f_old.close()
 384         file_out.close()
 385
 386
 387
 388 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 389         #Full File Path should MATCH
 390         if GenerateDiffAttr == "FALSE":
 391                 return
 392         FilePath = '"/'+RequestedPath+'"'
 393         #print ('FilePath - %s'% (FilePath))
 394         with open(AttributeFile) as f:
 395                 for line in f:
 396                         if FilePath in line:
 397                                 if Type == SYMLINK_TYPE:
 398                                         Sym_Attibutes.append(line)
 399                                 else:
 400                                         File_Attibutes.append(line)
 401
 402
 403 '''This function returns the SHA-1 hash of the file passed into it'''
 404 def hash_file(filename):
 405
 406    # make a hash object
 407    h = hashlib.sha1()
 408
 409    # open file for reading in binary mode
 410    with open(filename,'rb') as file:
 411        # loop till the end of the file
 412        chunk = 0
 413        while chunk != b'':
 414            # read only 1024 bytes at a time
 415            chunk = file.read(1024*1024)
 416            h.update(chunk)
 417
 418    # return the hex representation of digest
 419    return h.hexdigest()
 420
 421 def find_dupes_dir(BASE_OLD, BASE_NEW):
 422         dups = {}
 423         fdupes = {}
 424         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 425         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 426         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 427                 #print('Scanning %s...' % rootbase)
 428                 for filename in fileListB:
 429                         path = os.path.join(rootbase, filename)
 430                         if os.path.islink(path):
 431                                 continue
 432                         # Calculate hash
 433                         file_hash = hash_file(path)
 434                         dups[file_hash] = path
 435
 436         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 437                 #print('Scanning %s...' % roottarget)
 438                 for filename in fileListT:
 439                         # Get the path to the file
 440                         path = os.path.join(roottarget, filename)
 441                         if os.path.islink(path):
 442                                 continue
 443                         # Calculate hash
 444                         file_hash = hash_file(path)
 445                         # Add or append the file path
 446                         if file_hash in dups:
 447                                 BaseStr = dups.get(file_hash)
 448                                 Baseloc = path.find('/')
 449                                 TarLoc = BaseStr.find('/')
 450                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 451                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 452                                         fdupes[path] = BaseStr
 453         logging.info('Total Duplicate files %d' % (len(fdupes)))
 454         return fdupes
 455
 456
 457 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 458         dups = {}
 459         fdupes = {}
 460         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 461
 462         for filename in fileListB:
 463                 Src_File = BASE_OLD+'/'+filename
 464                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 465                         continue
 466                 # Calculate hash
 467                 file_hash = hash_file(Src_File)
 468                 dups[file_hash] = Src_File
 469
 470
 471         for filename in fileListT:
 472                 Dest_File = BASE_NEW+'/'+filename
 473                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 474                         continue
 475                 # Calculate hash
 476                 file_hash = hash_file(Dest_File)
 477                 if file_hash in dups:
 478                         BaseStr = dups.get(file_hash)
 479                         Baseloc = BaseStr.find('/')
 480                         if not BaseStr[Baseloc:] == filename:
 481                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 482                                 fdupes[BaseStr] = filename
 483
 484         logging.info('Total Duplicate files %d' % (len(fdupes)))
 485         return fdupes
 486
 487 def SS_UpdateSize(src_file, dst_file):
 488         global MEM_REQ
 489         global MEM_FILE
 490         oldsize_d= os.path.getsize(src_file)
 491         newsize_d= os.path.getsize(dst_file)
 492         if oldsize_d >= newsize_d:
 493                 Max = newsize_d
 494         else:
 495                 Max = oldsize_d
 496         if MEM_REQ < Max:
 497                 MEM_REQ = Max
 498                 MEM_FILE = dst_file
 499
 500
 501
 502 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 503         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 504         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 505
 506         # First let's fill up these categories
 507         files_new = []
 508         files_removed = []
 509         Dir_removed = []
 510         Dir_Added = []
 511         files_changed = []
 512         files_unchanged = []
 513         files_renamed = []
 514         File_Attibutes = []
 515         Sym_Attibutes = []
 516
 517         files_Del_List = {}
 518         files_New_List = {}
 519         MyDict_Patches = {}
 520
 521
 522
 523         PWD = os.getcwd()
 524
 525         # Generate NEW List
 526         for elt in New_files:
 527                 if elt not in Old_files:
 528                         files_new.append(elt)
 529                         logging.info('New files %s' % elt)
 530
 531         # Generate Delete List
 532         for elt in Old_files:
 533                 if elt not in New_files:
 534                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 535                         files_removed.append(elt)
 536                         logging.info('Old files %s' % elt)
 537
 538
 539         for elt in Old_dirs:
 540                 #print('List of Old Dirs %s' % elt)
 541                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 542                 if elt not in New_dirs:
 543                         Dir_removed.append(elt)
 544                         logging.info('Old Dirs %s' % elt+'/')
 545
 546         for elt in New_dirs:
 547                 if elt not in Old_dirs:
 548                         Dir_Added.append(elt)
 549                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 550
 551         # What files have changed contents but not name/path?
 552         for elt in New_files:
 553                 if elt in Old_files:
 554                         #Both are symbolic linkes and they differ
 555                         src_file = BASE_OLD+'/'+elt
 556                         dst_file = BASE_NEW+'/'+elt
 557                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 558                         if os.path.islink(src_file) and os.path.islink(dst_file):
 559                                 if not os.readlink(src_file) == os.readlink(dst_file):
 560                                         files_changed.append(elt)
 561                                         #print('%d Sym link files changed' % len(files_changed))
 562                                         logging.info('Sym links Changed - %s' % elt)
 563                                 else:
 564                                         files_unchanged.append(elt)
 565                         #Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 566                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 567                                 if not filecmp.cmp(src_file, dst_file):
 568                                         files_changed.append(elt)
 569                                         #print('%d Normal files changed' % len(files_changed))
 570                                         #print('Files Changed - %s' % elt)
 571                                 else:
 572                                         files_unchanged.append(elt)
 573                         #File types differ between BASE and TARGET
 574                         else:
 575                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 576                                 #Both file types have changed and they differ
 577                                 #Case 1: First Delete the OLD entry file type (Be it anything)
 578                                 #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 579                                 files_removed.append(elt)
 580                                 files_new.append(elt)
 581
 582
 583         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 584         #This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 585
 586         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 587         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 588                 with open(VERBATIM_LIST, 'r') as F_News:
 589                         lines = set(F_News.read().splitlines())
 590                 for line in lines:
 591                         if line in files_changed:
 592                                 files_changed.remove(line)
 593                                 files_removed.append(line)
 594                         if line in files_new:
 595                                 files_new.remove(line)
 596
 597         #Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 598         if SUPPORT_RENAME == "TRUE":
 599                 for elt in files_removed:
 600                         if os.path.isfile(BASE_OLD+'/'+elt):
 601                                 FileName = path_leaf(elt)
 602                                 entries = re.split('[0-9]' , FileName)
 603                                 #Gives the STRING part of NAME. if name starts with version then later part wil b string
 604                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 605                                 #If version is starting at the begining of the string?? shd we hav additional check for such cases??
 606                                 if len(entries[0]) > 0:
 607                                         files_Del_List.update({entries[0]: elt})
 608
 609                 for elt in files_new:
 610                         if os.path.isfile(BASE_NEW+'/'+elt):
 611                                 FileName = path_leaf(elt)
 612                                 entries = re.split('[0-9]' , FileName)
 613                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 614                                 if len(entries[0]) > 0:
 615                                         files_New_List.update({entries[0]: elt})
 616
 617                 for key, value in files_Del_List.iteritems():
 618                         #print('Key value pair -%s -%s' % (key, value))
 619                         if key in files_New_List:
 620                                 # this file is the same name in both!
 621                                 src_file = BASE_OLD+'/'+value
 622                                 dst_file = BASE_NEW+'/'+files_New_List[key]
 623                                 olddirpath = path_head(files_New_List[key])
 624                                 newdirpath = path_head(value)
 625                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 626                                         logging.debug('Cannot diff as one of them is Symlink')
 627                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 628                                         logging.debug('Cannot diff as one of them is dir')
 629                                 else:
 630                                         #Pick the best diff of same type and diff names
 631                                         files_renamed.append([files_New_List[key], value])
 632                                         files_removed.remove(value)
 633                                         files_new.remove(files_New_List[key])
 634
 635         '''
 636         Patch Section
 637                 Partition.txt contains Protocol for UPI
 638                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 639         '''
 640         Sym_Diff_Cnt = 0
 641         Sym_New_Cnt = 0;
 642         Del_Cnt = 0
 643         New_Cnt = 0
 644         Diff_Cnt = 0
 645         Move_Cnt = 0
 646         Verbatim_Cnt = 0
 647         SymLinkDoc = OUT_DIR+'/'+PART_NAME+SYMLINK_DOC_NAME
 648         Partition_Doc = open(OUT_DIR+'/'+PART_NAME+'.txt','w')
 649         Partition_Doc_SymLinks = open(SymLinkDoc,'w')
 650
 651         print("writing diff'ed changed files...")
 652         for elt in files_changed:
 653                 dst_file = BASE_NEW+'/'+elt
 654                 src_file = BASE_OLD+'/'+elt
 655                 #Both files are symbolic links and they differ
 656                 if os.path.islink(dst_file) and os.path.islink(src_file):
 657                 #Both are symlinks and they differ
 658                         logging.debug(' File Changed is Link %s ' % dst_file)
 659                         patch = os.readlink(dst_file)
 660                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 661                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 662                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 663                 #Both are NORMAL files and they differ
 664                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 665                         #Both are files and they differ
 666                         Diff_Cnt = Diff_Cnt + 1
 667                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 668                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 669                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 670                         SS_UpdateSize(src_file, dst_file)
 671                         if SUPPORT_CONTAINERS == "TRUE":
 672                                 if src_file.endswith('.zip') and dst_file.endswith('.zip'):
 673                                         FORMAT = "ZIP"
 674                                         Partition_Doc.write('DIFF:ZIP:%s:%s:%s:%s:%s/\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 675                                         compute_containerdelta(src_file, dst_file, FORMAT, OUT_DIR+'/'+patchName, Partition_Doc)
 676                                 elif src_file.endswith('.tpk') and dst_file.endswith('.tpk'):
 677                                         FORMAT = "TPK"
 678                                         Partition_Doc.write('DIFF:TPK:%s:%s:%s:%s:%s/\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 679                                         compute_containerdelta(src_file, dst_file, FORMAT, OUT_DIR+'/'+patchName, Partition_Doc)
 680                                 else:
 681                                         FORMAT = "REG"
 682                                         Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 683                                         subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 684                         else:
 685                                 FORMAT = "REG"
 686                                 ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 687                                 if ret is not 0:
 688                                         logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 689                                         files_new.append(elt)
 690                                         Diff_Cnt = Diff_Cnt - 1
 691                                 else:
 692                                         Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 693
 694                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 695                 #Both differ but they are of diff types
 696                 else:
 697                         #Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 698                         files_removed.append(elt)
 699                         files_new.append(elt)
 700
 701         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 702         for oldpath, newpath in fdupes.iteritems():
 703                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 704
 705         for elt in files_removed:
 706                 src_file = BASE_OLD+'/'+elt
 707                 #If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 708                 if src_file in fdupes.keys():
 709                         dst_file = BASE_NEW+'/'+ fdupes[src_file]
 710                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 711                         Move_Cnt = Move_Cnt + 1
 712                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 713                         files_removed.remove(elt)
 714                         files_new.remove(fdupes[src_file])
 715
 716         #Should be placed after removing duplicates, else they will be filtered here.
 717         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 718         #       First Step: Sort & Filter out unwanted files
 719         # Minimum condition used is,
 720         #       1. File name should match 70%
 721         #       2. Extensions should be same
 722         #       3. File name length shd b greater than 3 char
 723         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 724         #   5. Should consider editdistance for RENAME LOGIC ==> TBD
 725
 726         Base_DelList = files_removed[:]
 727         Base_NewList = files_new[:]
 728         DelList = sorted(Base_DelList, key=path_leaf)
 729         NewList = sorted(Base_NewList, key=path_leaf)
 730         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 731
 732         Filter1 = []
 733         Filter2 = []
 734         #Remove unwanted items which we cant make diff with for rename logic
 735         for file in DelList:
 736                 if os.path.islink(BASE_OLD+'/'+file):
 737                         continue
 738                 elif os.path.isdir(BASE_OLD+'/'+file):
 739                         continue
 740                 else:
 741                         Filter1.append(file)
 742                         #logging.debug('Sorted del list - %s' % (file))
 743
 744         DelList = Filter1
 745
 746         for file in NewList:
 747                 if os.path.islink(BASE_NEW+'/'+file):
 748                         continue
 749                 elif os.path.isdir(BASE_NEW+'/'+file):
 750                         continue
 751                 elif len(path_leaf(file)) <= 3:
 752                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW+'/'+file))
 753                         continue
 754                 else:
 755                         Filter2.append(file)
 756
 757         NewList = Filter2
 758
 759         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 760
 761         for new_file in NewList:
 762                 R_Flag = 'FALSE';
 763                 DirPathNew = path_head(new_file)
 764                 FileNameNew = path_leaf(new_file)
 765                 DiffSize = 0
 766                 winning_patch_sz = os.path.getsize(BASE_NEW+'/'+new_file)
 767                 New_fs = winning_patch_sz
 768                 winning_file = ''
 769
 770                 for del_file in DelList:
 771                         FileNameOld = path_leaf(del_file)
 772                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 773                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 774                                 #Percentage difference between two file sizes is within 30%, then we consider for diff generation
 775                                 Del_fs = os.path.getsize(BASE_OLD+'/'+del_file)
 776                                 v1 = abs(New_fs-Del_fs)
 777                                 v2 = (New_fs+Del_fs)/2
 778                                 if( v2<=0 or ((v1/v2) * 100) > 30 ):
 779                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 780                                         continue
 781                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 782                                 R_Flag = 'TRUE';
 783                                 DiffSize = measure_two_filediffs(BASE_OLD+'/'+del_file, BASE_NEW+'/'+new_file)
 784                                 if (DiffSize < 0.8 * winning_patch_sz):
 785                                         winning_patch_sz = DiffSize
 786                                         winning_file = del_file
 787                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew)*7/10]) and R_Flag == 'TRUE'):
 788                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 789                                 break;
 790                 if len(winning_file) > 0:
 791                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 792                         files_renamed.append([new_file, winning_file])
 793                         DelList.remove(winning_file)
 794                         files_removed.remove(winning_file)
 795                         files_new.remove(new_file)
 796
 797         #********************** Files should NOT be deleted for any such renames ***********************
 798
 799         if SUPPORT_RENAME == "TRUE":
 800                 for elt in files_renamed:
 801                         src_file = BASE_OLD+'/'+elt[1]
 802                         dst_file = BASE_NEW+'/'+elt[0]
 803                         Diff_Cnt = Diff_Cnt + 1
 804                         patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 805                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 806                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 807                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 808                         # Should be careful of renaming files??
 809                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 810                         # What if folder path has numerics??
 811
 812                         if  os.path.isdir(src_file) or os.path.isdir(dst_file):
 813                                 #This case never occurs??
 814                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 815                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 816                         else: #Make sure these files are PROPER and they shd NOT be symlinks
 817                                 if filecmp.cmp(src_file, dst_file):
 818                                         Move_Cnt = Move_Cnt + 1
 819                                         Diff_Cnt = Diff_Cnt - 1
 820                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 821                                 elif SUPPORT_CONTAINERS == "TRUE":
 822                                         if src_file.endswith('.zip') and dst_file.endswith('.zip'):
 823                                                 FORMAT = "ZIP"
 824                                                 Partition_Doc.write('DIFF:ZIP:%s:%s:%s:%s:%s/\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 825                                                 compute_containerdelta(src_file, dst_file, FORMAT, OUT_DIR+'/'+patchName, Partition_Doc)
 826                                         elif src_file.endswith('.tpk') and dst_file.endswith('.tpk'):
 827                                                 FORMAT = "TPK"
 828                                                 Partition_Doc.write('DIFF:TPK:%s:%s:%s:%s:%s/\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 829                                                 compute_containerdelta(src_file, dst_file, FORMAT, OUT_DIR+'/'+patchName, Partition_Doc)
 830                                         else:
 831                                                 FORMAT = "REG"
 832                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 833                                                 subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 834                                 else:
 835                                         FORMAT = "REG"
 836                                         ret = subprocess.call([DIFF_UTIL,src_file,dst_file,patchLoc])
 837                                         if ret is not 0:
 838                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 839                                                 files_new.append(elt)
 840                                                 Diff_Cnt = Diff_Cnt - 1
 841                                         else:
 842                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 843
 844                                 SS_UpdateSize(src_file, dst_file)
 845                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 846
 847
 848         #HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 849         #So that, the rename functionality will NOT create PATCH instead of verbatims.
 850
 851         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 852                 with open(VERBATIM_LIST, 'r') as F_News:
 853                         lines = set(F_News.read().splitlines())
 854                 for line in lines:
 855                         if not line in files_new:
 856                                 if os.path.exists(BASE_NEW+'/'+line):
 857                                         files_new.append(line)
 858                                         Verbatim_Cnt = Verbatim_Cnt+1
 859                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW+'/'+line)
 860
 861
 862
 863         for elt in files_removed:
 864                 #if files are part of patches after renaming, we shd remove them as part of removed.
 865                 src_file = BASE_OLD+'/'+elt
 866                 if os.path.islink(src_file):
 867                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 868                 elif os.path.isdir(src_file):
 869                         #If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 870                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 871                 else:
 872                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 873                 logging.debug(' File Deleted %s' % src_file)
 874                 Del_Cnt = Del_Cnt + 1
 875
 876         Dir_removed.sort(reverse=True)
 877         for elt in Dir_removed:
 878                 #if Dir is empty, add it to the removed list.
 879                 src_file = BASE_OLD+'/'+elt
 880                 #Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 881                 #It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 882                 if os.path.isdir(src_file):
 883                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 884                         Del_Cnt = Del_Cnt + 1
 885                         logging.debug(' Dir Deleted- %s' % src_file)
 886
 887
 888         for elt in files_new:
 889                 dst_file = BASE_NEW+'/'+elt
 890                 newfiles_dest_path = 'system/'
 891                 ensure_dir_exists(newfiles_dest_path)
 892                 if os.path.islink(dst_file):
 893                         patch = os.readlink(dst_file)
 894                         logging.debug(' File New Links %s' % elt)
 895                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 896                         #What if this is only a new sym link and folder already exists??? Should recheck
 897                         destpath = newfiles_dest_path + elt
 898                         if not os.path.exists(path_head(destpath)):
 899                                 os.makedirs(path_head(destpath))
 900                                 logging.info('New SymLink - Adding missing Dir')
 901                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 902                         Sym_New_Cnt = Sym_New_Cnt + 1
 903                 elif os.path.isdir(dst_file): # We create just empty directory here
 904                         destpath = newfiles_dest_path + elt
 905                         if not os.path.exists(destpath):
 906                                 os.makedirs(destpath)
 907                                 logging.debug(' File New Dir %s' % destpath)
 908                                 New_Cnt = New_Cnt + 1
 909                 else:
 910                         New_Cnt = New_Cnt + 1
 911                         #newfiles_dest_path = OUT_DIR + '/system/'
 912                         destpath = newfiles_dest_path + elt
 913                         destdir = os.path.dirname(destpath)
 914                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 915
 916                         if not os.path.isdir(destdir):
 917                                 try:
 918                                         os.makedirs(destdir)
 919                                 except:
 920                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 921                                         raise
 922
 923                         try:
 924                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 925                                         shutil.copy2(dst_file, destpath)
 926                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 927                         except:
 928                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 929                                 raise
 930
 931         for elt in Dir_Added:
 932                 newfiles_dest_path = 'system/'
 933                 ensure_dir_exists(newfiles_dest_path)
 934                 destpath = newfiles_dest_path + elt
 935                 if not os.path.exists(destpath):
 936                         os.makedirs(destpath)
 937                         logging.debug(' DirList New Dir %s' % destpath)
 938                         New_Cnt = New_Cnt + 1
 939
 940         #Base directory should be system
 941         print 'Compressing New files'
 942         if (New_Cnt > 0):
 943                 WorkingDir = os.getcwd()
 944                 os.chdir(os.getcwd()+"/"+NEW_FILES_PATH)
 945                 logging.info('Curr Working Dir - %s' % os.getcwd())
 946                 os.system(ZIPUTIL+NEW_FILES_PATH+" >> " + LOGFILE)
 947                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR)
 948                 #New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 949                 SS_UpdateSize(WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME, WorkingDir+"/"+OUT_DIR+"/"+NEW_FILES_ZIP_NAME)
 950                 os.chdir(WorkingDir)
 951                 shutil.rmtree(NEW_FILES_PATH)
 952                 # use 7z a system.7z ./*
 953
 954         #logging.info('%d Dir to be removed' % len(Dir_removed))
 955         logging.info('%d files unchanged' % len(files_unchanged))
 956         logging.info('%d files files_renamed' % len(files_renamed))
 957         logging.info('%d files NEW' % len(files_new))
 958         logging.info('%d File attr' % len(File_Attibutes))
 959         logging.info('%d Sym attr' % len(Sym_Attibutes))
 960         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 961         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 962
 963         #There could be duplicates, TODO, can check before adding..
 964         ATTR_FILE_D = open(ATTR_FILE,'a+')
 965         for elt in File_Attibutes:
 966                 ATTR_FILE_D.write(elt)
 967         for elt in Sym_Attibutes:
 968                 ATTR_FILE_D.write(elt)
 969
 970         ATTR_FILE_D.close()
 971
 972         Partition_Doc_SymLinks.close()
 973         Partition_Read_SymLinks = open(SymLinkDoc,'r+')
 974         Partition_Doc.write(Partition_Read_SymLinks.read())
 975         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 976         Partition_Doc_SymLinks.close()
 977         Partition_Doc.close()
 978         os.remove(SymLinkDoc)
 979
 980         if Diff_Cnt + Move_Cnt + New_Cnt+ Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 981                 print('No Delta Generated for %s - %s' % (PART_NAME,OUT_DIR))
 982                 logging.info('No Delta Generated for %s' % PART_NAME)
 983                 shutil.rmtree(OUT_DIR)
 984
 985
 986 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 987
 988         #CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 989         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 990         shutil.copy2(g_output_dir+'/'+b_apk, g_output_dir+'/temp')
 991         temp_apk = '../'+g_output_dir+'/'+b_apk
 992         Patch = 'Patch_'+b_apk
 993         ensure_dir_exists(Patch)
 994         shutil.copy2(g_output_dir+'/'+b_apk, Patch+'/'+b_apk)
 995
 996         #Size issue on Device side?? shd check this
 997         subprocess.call(['unzip','-q', Patch+'/'+b_apk, '-d', Patch])
 998         with open(g_output_dir+'/PATCH.txt', 'r') as f_new:
 999                 lines = set(f_new.read().splitlines())
1000                 for line in lines:
1001                         #print('Action ==> %s' % line)
1002                         #Action, Path, Patch = line.split('|')
1003                         Items = line.split('|')
1004                         Action = Items[0]
1005                         Path = Items[1]
1006                         ActualPath = a_folder+'/'+Path
1007                         PatchPath = Patch+'/'+Path
1008                         SrcPath = g_output_dir+'/'+path_leaf(Path)
1009                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
1010                         if line[0] == 'c':
1011                                 patchName = g_output_dir+'/'+Items[2]
1012                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
1013                                 subprocess.call([DIFFPATCH_UTIL,ActualPath,ActualPath,patchName])
1014                                 WorkingDir = os.getcwd()
1015                                 os.chdir(WorkingDir+"/"+"temp_a")
1016                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
1017                                 os.chdir(WorkingDir)
1018                         elif line[0] == 's':
1019                                 WorkingDir = os.getcwd()
1020                                 os.chdir(WorkingDir+"/"+"temp_a")
1021                                 subprocess.call(['cp', '--parents', Path, '../'+Patch])
1022                                 os.chdir(WorkingDir)
1023                         else:
1024                                 print('Apply_Container_Delta - Unknown Error')
1025         #print('Touch all files and set common attributes for DIFF generation')
1026         WorkingDir = os.getcwd()
1027         os.chdir(WorkingDir+"/"+Patch)
1028
1029         CONTAINER_DATE = '200011111111.11'
1030         CONTAINER_MODE = '0755'
1031         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
1032         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
1033         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../'+Patch])
1034
1035         print 'Update Intermediate Archive'
1036         #subprocess.call(['zip','-ryX', b_apk, '*'])
1037         subprocess.call(['zip','-ryX', b_apk] + glob.glob('*'))
1038         os.chdir(WorkingDir)
1039         #print('Apply Path completed - Now create diff for this and place in patch folder')
1040         #print os.getcwd()
1041         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir+'/'+b_apk,new_apk))
1042         patchName = ('New'+'_%s'+DIFF_SUFFIX) % (b_apk)
1043         patchLoc = '%s/%s' % (g_output_dir, patchName)
1044
1045         subprocess.call([DIFF_UTIL, Patch+'/'+b_apk ,new_apk,patchLoc])
1046
1047         #Only on HOST... for testing
1048         if TEST_MODE == 'TRUE':
1049                 UpgradedName = '%s_Upgraded' % (b_apk)
1050                 subprocess.call([DIFFPATCH_UTIL,Patch+'/'+b_apk,UpgradedName,patchLoc])
1051
1052         #This is file only with NEWS and empty diffs and same files.
1053         if TEST_MODE == 'FALSE':
1054                 os.remove(g_output_dir+'/'+b_apk)
1055                 os.rename(g_output_dir+'/temp', g_output_dir+'/'+b_apk)
1056                 shutil.rmtree(Patch)
1057
1058 def IsSymlink(info):
1059   return (info.external_attr >> 16) == 0120777
1060
1061
1062 def compute_containerdelta(src_file, dst_file, FORMAT, patchName, Partition_Doc):
1063
1064         a_apk = src_file
1065         b_apk = dst_file
1066         a_folder = 'temp_a'
1067         b_folder = 'temp_b'
1068
1069         g_output_dir = patchName
1070
1071         logging.info('Uncompressing Containers... [%s][%s]' % (src_file, dst_file))
1072         logging.info('Out Dir -%s' %(g_output_dir))
1073         ensure_dir_exists(a_folder)
1074         zipf = zipfile.ZipFile(a_apk, 'r');
1075         zipf.extractall(a_folder)
1076         zipf.close()
1077
1078         ensure_dir_exists(b_folder)
1079         zipf = zipfile.ZipFile(b_apk, 'r');
1080         zipf.extractall(b_folder)
1081
1082
1083         symlinks = []
1084         for info in zipf.infolist():
1085                 basefilename = info.filename[7:]
1086                 if IsSymlink(info):
1087                         symlinks.append(info.filename)
1088                         os.remove(b_folder+'/'+info.filename)
1089         zipf.close()
1090
1091         a_files, a_dirs = Get_Files(a_folder)
1092         b_files, b_dirs = Get_Files(b_folder)
1093
1094         logging.info('Going from %d files %d files' % (len(a_files), len(b_files)))
1095
1096         # First let's fill up these categories
1097         C_files_new = []
1098         C_files_removed = []
1099         C_files_changed = []
1100         C_files_unchanged = []
1101
1102         # What files appear in B but not in A?
1103         for elt in b_files:
1104                 if elt not in a_files:
1105                         #if not elt.endswith('.so'):
1106                         C_files_new.append(elt)
1107
1108         # What files appear in A but not in B?
1109         for elt in a_files:
1110                 if elt not in b_files:
1111                         C_files_removed.append(elt)
1112
1113         # What files have changed contents but not name/path?
1114         for elt in b_files:
1115                 if elt in a_files:
1116                         if os.path.islink(a_folder+'/'+elt) or os.path.islink(b_folder+'/'+elt):
1117                                 print 'links - skip'
1118                         elif not filecmp.cmp(a_folder+'/'+elt, b_folder+'/'+elt):
1119                                 C_files_changed.append(elt)
1120                         else:
1121                                 C_files_unchanged.append(elt)
1122
1123
1124         print('%d new files' % len(C_files_new))
1125         print('%d removed files' % len(C_files_removed))
1126         print('%d files changed' % len(C_files_changed))
1127         print('%d files unchanged' % len(C_files_unchanged))
1128
1129         # temp dir where we're assembling the patch
1130         ensure_dir_exists(g_output_dir)
1131
1132         unique_fileid = 0
1133         toc = open(g_output_dir+'/PATCH.txt','w')
1134         print("writing diff'ed changed files...")
1135
1136         for elt in C_files_changed:
1137                 dst_file = b_folder+'/'+elt
1138                 src_file = a_folder+'/'+elt
1139                 patchName = (DIFF_PREFIX+'%d_%s_'+PART_NAME+DIFF_SUFFIX) % (unique_fileid, path_leaf(elt))
1140                 patchLoc = '%s/%s' % (g_output_dir, patchName)
1141                 #print('src - %s dest -%s patch -%s' % (src_file ,dst_file,patchLoc))
1142                 subprocess.call([DIFF_UTIL,src_file ,dst_file,patchLoc])
1143                 toc.write('c%d|%s|%s\n' % (unique_fileid, elt, patchName))
1144                 unique_fileid = unique_fileid + 1
1145
1146         for elt in C_files_unchanged:
1147                 dst_file = b_folder+'/'+elt
1148                 src_file = a_folder+'/'+elt
1149                 #print('Same Files src - %s dest -%s' % (src_file ,dst_file))
1150                 toc.write('s%d|%s\n' % (unique_fileid, elt))
1151                 unique_fileid = unique_fileid + 1
1152
1153         #Create NEW TPK with empty data for below files and NEW files
1154         shutil.copy2(b_apk, g_output_dir)
1155
1156         #May b for host??
1157         #temp_apk = '../'+g_output_dir+'/'+b_apk
1158         temp_apk = '../'+g_output_dir+'/'+path_leaf(b_apk)
1159
1160         for elt in C_files_changed:
1161                 dst_file = b_folder+'/'+elt
1162                 #print dst_file
1163                 open(dst_file, 'w').close()
1164
1165         for elt in C_files_unchanged:
1166                 dst_file = b_folder+'/'+elt
1167                 open(dst_file, 'w').close()
1168
1169         WorkingDir = os.getcwd()
1170         os.chdir(WorkingDir+"/"+b_folder)
1171
1172         #for elt in files_changed:
1173         #       subprocess.call(['zip', temp_apk, elt]) # confirm ZIP options, extra fields etc.. jus zip it, shd do all at once.. else time taking
1174
1175         #for elt in files_unchanged:
1176         #       subprocess.call(['zip', temp_apk, elt])
1177
1178         subprocess.call(['zip','-ryq', temp_apk, '*'])
1179         os.chdir(WorkingDir)
1180         toc.close()
1181
1182         Apply_Container_Delta(path_leaf(a_apk), path_leaf(b_apk), b_apk, a_folder, g_output_dir)
1183         shutil.rmtree(a_folder)
1184         shutil.rmtree(b_folder)
1185
1186 def NewFiles(src, dest):
1187         print src,dest
1188         subprocess.call(['cp','-rp', src,dest])
1189     #try:
1190                 #shutil.copytree(src, dest)
1191     #except OSError as e:
1192         # If the error was caused because the source wasn't a directory
1193         #if e.errno == errno.ENOTDIR:
1194             #shutil.copy2(src, dest)
1195         #else:
1196             #print('Directory not copied. Error: %s' % e)
1197
1198 def measure_two_filediffs(src, dst):
1199         patchLoc = 'temp.patch'
1200         subprocess.call([DIFF_UTIL,src,dst,patchLoc])
1201         result_size = os.path.getsize(patchLoc)
1202         os.remove(patchLoc)
1203         return result_size
1204
1205 def Get_Files(path):
1206         all_files = []
1207         all_dirs = []
1208
1209         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1210                 for directory in directories:
1211                         #DirName = os.path.join(root+'/',directory)
1212                         DirName = os.path.join(root,directory)
1213                         if os.path.islink(DirName):
1214                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1215                                 all_files.append(os.path.relpath(DirName, path))
1216                         elif not os.listdir(DirName):
1217                                 #print('*****Empty Directory******* -%s', DirName)
1218                                 #This should NOT be appended ??? Empty dir shd b considered
1219                                 all_dirs.append(os.path.relpath(DirName, path))
1220                         else:
1221                                 all_dirs.append(os.path.relpath(DirName, path))
1222                 for filename in filenames:
1223                         FileName = os.path.join(root,filename)
1224                         all_files.append(os.path.relpath(FileName, path))
1225
1226         all_files.sort()
1227         all_dirs.sort()
1228         return all_files, all_dirs
1229
1230
1231 USAGE_DOCSTRING = """
1232       Generate Delta using BASEOLD AND BASE NEW
1233           Attributes is optional
1234           Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1235 """
1236
1237 def Usage(docstring):
1238   print docstring.rstrip("\n")
1239   print COMMON_DOCSTRING
1240
1241
1242
1243 if __name__ == '__main__':
1244         main()
1245