mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49
  50 def global_paths():
  51         global DIFF_UTIL
  52         global ZIPUTIL
  53         global NEW_FILES_PATH
  54         global NEW_FILES_ZIP_NAME
  55         global SYMLINK_TYPE
  56         global ATTR_DOC_EXT
  57         global SYMLINK_DOC_NAME
  58         global DIFF_PREFIX
  59         global DIFF_SUFFIX
  60         global SUPPORT_RENAME
  61         global NEW_PREFIX
  62         global DIFFPATCH_UTIL
  63         global SUPPORT_CONTAINERS
  64         global FULL_IMAGE
  65         global DELTA_IMAGE
  66         global DELTA_FS
  67         global EXTRA
  68         global COMMON_BIN_PATH
  69         global MEM_REQ
  70         global EMPTY
  71         global VERBATIM_LIST
  72         global MEM_FILE
  73
  74
  75 COMMON_BIN_PATH = "../../common/bin/"
  76 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  77 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  78 #ZIPUTIL = "p7zip "
  79 ZIPUTIL = "7z -mf=off a system.7z "
  80 NEW_FILES_PATH = "run/upgrade-sysroot"
  81 NEW_FILES_ZIP_NAME = "system.7z"
  82 SYMLINK_TYPE = "SYM"
  83 ATTR_DOC_EXT = "_attr.txt"
  84 SYMLINK_DOC_NAME = "_sym.txt"
  85 PART_DOC_EXT = ".txt"
  86 DIFF_PREFIX = "diff"
  87 DIFF_SUFFIX = ".delta"
  88 NEW_PREFIX = 'new'
  89 FULL_IMAGE = "FULL_IMAGE"
  90 DELTA_IMAGE = "DELTA_IMAGE"
  91 DELTA_FS = "DELTA_FS"
  92 EXTRA = "EXTRA"
  93 LOGFILE = "Delta.log"
  94 VERBATIM_LIST = "Verbatim_List.txt"
  95 EMPTY = ""
  96 MEM_REQ = 0
  97 MEM_FILE = "NULL"
  98 COMPRESSION_LZMA = "lzma"
  99 COMPRESSION_BROTLI = "brotli"
 100
 101 SUPPORT_RENAME = "TRUE"  # Use appropriate name
 102 SUPPORT_CONTAINERS = "FALSE"
 103 SUPPORT_VERBATIM = "TRUE"
 104
 105 TEST_MODE = "FALSE"
 106
 107
 108 def main():
 109         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 110         global AttributeFile
 111         global GenerateDiffAttr
 112         try:
 113
 114                 if len(sys.argv) < 5:
 115                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 116                 UPDATE_TYPE = sys.argv[1]
 117                 UPDATE_TYPE_S = UPDATE_TYPE.split(":")
 118                 PART_NAME = sys.argv[2]  # lets make this also optional
 119
 120                 BASE_OLD = sys.argv[3]
 121                 BASE_NEW = sys.argv[4]
 122                 OUT_DIR = sys.argv[5]
 123                 ATTR_OLD = EMPTY
 124                 ATTR_NEW = EMPTY
 125                 UPDATE_CFG_PATH = EMPTY
 126                 GenerateDiffAttr = "FALSE"
 127                 if UPDATE_TYPE_S[0] == DELTA_FS:
 128                         #instead of arguments check it in outdirectory ?
 129                         if len(sys.argv) == 9:
 130                                 ATTR_OLD = sys.argv[6]
 131                                 ATTR_NEW = sys.argv[7]
 132                                 UPDATE_CFG_PATH = '../' + sys.argv[8]
 133                                 GenerateDiffAttr = "TRUE"
 134
 135                 elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]:
 136                         if len(sys.argv) == 7:
 137                                 #Use path in better way
 138                                 UPDATE_CFG_PATH = '../' + sys.argv[6]
 139
 140                 global DIFF_UTIL
 141                 global DIFFPATCH_UTIL
 142                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 143                         DIFF_UTIL = COMMON_BIN_PATH + DIFF_UTIL
 144                         DIFFPATCH_UTIL = COMMON_BIN_PATH + DIFFPATCH_UTIL
 145                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 146                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 147                                 logging.info('Diff Util Does NOT exist -- ABORT')
 148                                 sys.exit(1)
 149
 150                 start = datetime.datetime.now().time()
 151                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 152                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]' % (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 153
 154                 try:
 155                         ensure_dir_exists(OUT_DIR)
 156                 except FileExistsError as exc:
 157                         logging.error('Argument passed as OUT_DIR - %s is already an existing file' % OUT_DIR)
 158                         raise exc
 159                 if GenerateDiffAttr == "TRUE":
 160                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 161                                 print >> sys.stderr, "Attributes missing -- ABORT"
 162                                 sys.exit(1)
 163
 164                 # Should check if APT is supported on other linux flavours
 165                 cache = apt.Cache()
 166                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 167                         logging.info('Basic utils installed')
 168                 else:
 169                         print >> sys.stderr, "Basic utils missing -- ABORT"
 170                         sys.exit(1)
 171
 172                 if UPDATE_TYPE_S[0] == FULL_IMAGE:
 173                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 174                 # #### currently does not support LZMA ####
 175                 #  elif UPDATE_TYPE == DELTA_IMAGE:
 176                 #      SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
 177                 elif UPDATE_TYPE_S[0] == DELTA_IMAGE:
 178                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
 179                 elif UPDATE_TYPE == DELTA_FS:
 180                         AttributeFile = ATTR_NEW
 181                         ATTR_FILE = OUT_DIR + '/' + PART_NAME + ATTR_DOC_EXT
 182                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 183                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 184                         New_files, New_dirs = Get_Files(BASE_NEW)
 185                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 186
 187                         if not UPDATE_CFG_PATH == EMPTY:
 188                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 189
 190                 elif UPDATE_TYPE == EXTRA:
 191                         print('UPDATE_TYPE ---- EXTRA')
 192                 else:
 193                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 194
 195                 if GenerateDiffAttr == "TRUE":
 196                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 197                                 os.remove(ATTR_OLD)
 198                                 os.remove(ATTR_NEW)
 199                 end = datetime.datetime.now().time()
 200
 201                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 202                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 203                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 204                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 205
 206         except Exception as exc:
 207                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 208                 raise exc
 209
 210
 211 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 212         f = open(UPDATE_CFG_PATH, 'r')
 213         lines = f.readlines()
 214         f.close()
 215         f = open(UPDATE_CFG_PATH, 'w')
 216         for line in lines:
 217                 ConfigItems = line.split()
 218                 if ConfigItems[0] == DELTA_BIN:
 219                         DELTA = ConfigItems[1]
 220                         logging.info('Updating %s config' % DELTA_BIN)
 221                         line = line.rstrip('\n')
 222                         Value = MEM_REQ
 223                         line = line.replace(line, line + '\t' + str(Value) + '\n')
 224                         f.write(line)
 225                 else:
 226                         f.write(line)
 227         f.close()
 228
 229
 230 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
 231         #for sizes
 232
 233         oldsize_d = os.path.getsize(BASE_OLD)
 234         newsize_d = os.path.getsize(BASE_NEW)
 235         SHA_BIN_DEST = hash_file(BASE_NEW)
 236         SHA_BIN_BASE = hash_file(BASE_OLD)
 237
 238         #incase UPDATE CFG is empty
 239         DELTA = DELTA_BIN
 240         SS_UpdateSize(BASE_OLD, BASE_NEW)
 241         #Should throw error if PART NAME NOT found??
 242         if not UPDATE_CFG_PATH == EMPTY:
 243                 f = open(UPDATE_CFG_PATH, 'r')
 244                 lines = f.readlines()
 245                 f.close()
 246                 f = open(UPDATE_CFG_PATH, 'w')
 247                 for line in lines:
 248                         ConfigItems = line.split()
 249                         if ConfigItems[0] == DELTA_BIN:
 250                                 logging.info('Updating %s config' % DELTA_BIN)
 251                                 DELTA = ConfigItems[1]
 252                                 line = line.rstrip('\n')
 253                                 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
 254                                 f.write(line)
 255                         else:
 256                                 f.write(line)
 257                 f.close()
 258
 259         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 260         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW, DELTA_BIN, patchLoc))
 261         subprocess.call([DIFF_UTIL, "-c", COMPRESSION_METHOD, BASE_OLD, BASE_NEW, patchLoc])
 262
 263
 264 def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 265         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW, DELTA_BIN))
 266         oldsize_d = os.path.getsize(BASE_OLD)
 267         newsize_d = os.path.getsize(BASE_NEW)
 268         SHA_BIN_DEST = hash_file(BASE_NEW)
 269         SHA_BIN_BASE = hash_file(BASE_OLD)
 270         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 271         SS_UpdateSize(BASE_OLD, BASE_NEW)
 272
 273         if not UPDATE_CFG_PATH == EMPTY:
 274                 f = open(UPDATE_CFG_PATH, 'r')
 275                 lines = f.readlines()
 276                 f.close()
 277                 f = open(UPDATE_CFG_PATH, 'w')
 278                 for line in lines:
 279                         ConfigItems = line.split()
 280                         if ConfigItems[0] == DELTA_BIN:
 281                                 logging.info('Updating %s config' % DELTA_BIN)
 282                                 DELTA = ConfigItems[1]
 283                                 line = line.rstrip('\n')
 284                                 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
 285                                 f.write(line)
 286                         else:
 287                                 f.write(line)
 288                 f.close()
 289
 290
 291 def zipdir(path, zip):
 292         for root, dirs, files in os.walk(path):
 293                 for file in files:
 294                         zip.write(os.path.join(root, file))
 295
 296
 297 def ensure_dir_exists(path):
 298         if not os.path.exists(path):
 299                 os.makedirs(path)
 300         elif os.path.isfile(path):
 301                 raise FileExistsError
 302                 #shutil.rmtree(path)
 303         #os.makedirs(path)
 304
 305
 306 def path_leaf(path):
 307         head, tail = ntpath.split(path)  # This is for windows?? Recheck
 308         return tail
 309
 310
 311 def path_head(path):
 312         head, tail = ntpath.split(path)
 313         return head
 314
 315
 316 # Creating Diff between OLD and NEW attribute files v12
 317 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 318         if GenerateDiffAttr == "FALSE":
 319                 return
 320         with open(ATTR_OLD, 'r') as f_old:
 321                 lines1 = set(f_old.read().splitlines())
 322
 323         with open(ATTR_NEW, 'r') as f_new:
 324                 lines2 = set(f_new.read().splitlines())
 325
 326         lines = set.difference(lines2, lines1)
 327         with open(ATTR_FILE, 'w+') as file_out:
 328                 for line in lines:
 329                         logging.info('Diff_AttrFiles - %s' % line)
 330                         file_out.write(line + '\n')
 331
 332
 333 def Update_Attr(RequestedPath, Type, File_Attibutes, Sym_Attibutes):
 334         # Full File Path should MATCH
 335         if GenerateDiffAttr == "FALSE":
 336                 return
 337         FilePath = '"/' + RequestedPath + '"'
 338         #print ('FilePath - %s'% (FilePath))
 339         with open(AttributeFile) as f:
 340                 for line in f:
 341                         if FilePath in line:
 342                                 if Type == SYMLINK_TYPE:
 343                                         Sym_Attibutes.append(line)
 344                                 else:
 345                                         File_Attibutes.append(line)
 346
 347
 348 def hash_file(filename):
 349         '''This function returns the SHA-1 hash of the file passed into it'''
 350
 351         # make a hash object
 352         h = hashlib.sha1()
 353
 354         # open file for reading in binary mode
 355         with open(filename, 'rb') as file:
 356                 # loop till the end of the file
 357                 chunk = 0
 358                 while chunk != b'':
 359                         # read only 1024 bytes at a time
 360                         chunk = file.read(1024 * 1024)
 361                         h.update(chunk)
 362
 363         # return the hex representation of digest
 364         return h.hexdigest()
 365
 366
 367 def find_dupes_dir(BASE_OLD, BASE_NEW):
 368         dups = {}
 369         fdupes = {}
 370         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 371         logging.info('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 372         for rootbase, subdirsB, fileListB in os.walk(BASE_OLD):
 373                 #print('Scanning %s...' % rootbase)
 374                 for filename in fileListB:
 375                         path = os.path.join(rootbase, filename)
 376                         if os.path.islink(path):
 377                                 continue
 378                         # Calculate hash
 379                         file_hash = hash_file(path)
 380                         dups[file_hash] = path
 381
 382         for roottarget, subdirsT, fileListT in os.walk(BASE_NEW):
 383                 #print('Scanning %s...' % roottarget)
 384                 for filename in fileListT:
 385                         # Get the path to the file
 386                         path = os.path.join(roottarget, filename)
 387                         if os.path.islink(path):
 388                                 continue
 389                         # Calculate hash
 390                         file_hash = hash_file(path)
 391                         # Add or append the file path
 392                         if file_hash in dups:
 393                                 BaseStr = dups.get(file_hash)
 394                                 Baseloc = path.find('/')
 395                                 TarLoc = BaseStr.find('/')
 396                                 if not path[Baseloc:] == BaseStr[TarLoc:]:
 397                                         logging.info('Dupes - %s ==> %s' % (path[Baseloc:], BaseStr[TarLoc:]))
 398                                         fdupes[path] = BaseStr
 399         logging.info('Total Duplicate files %d' % (len(fdupes)))
 400         return fdupes
 401
 402
 403 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT):
 404         dups = {}
 405         fdupes = {}
 406         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 407
 408         for filename in fileListB:
 409                 Src_File = BASE_OLD + '/' + filename
 410                 if os.path.islink(Src_File) or os.path.isdir(Src_File):
 411                         continue
 412                 # Calculate hash
 413                 file_hash = hash_file(Src_File)
 414                 dups[file_hash] = Src_File
 415
 416         for filename in fileListT:
 417                 Dest_File = BASE_NEW + '/' + filename
 418                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File):
 419                         continue
 420                 # Calculate hash
 421                 file_hash = hash_file(Dest_File)
 422                 if file_hash in dups:
 423                         BaseStr = dups.get(file_hash)
 424                         Baseloc = BaseStr.find('/')
 425                         if not BaseStr[Baseloc:] == filename:
 426                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 427                                 fdupes[BaseStr] = filename
 428
 429         logging.info('Total Duplicate files %d' % (len(fdupes)))
 430         return fdupes
 431
 432
 433 def SS_UpdateSize(src_file, dst_file):
 434         global MEM_REQ
 435         global MEM_FILE
 436         oldsize_d = os.path.getsize(src_file)
 437         newsize_d = os.path.getsize(dst_file)
 438         if oldsize_d >= newsize_d:
 439                 Max = newsize_d
 440         else:
 441                 Max = oldsize_d
 442         if MEM_REQ < Max:
 443                 MEM_REQ = Max
 444                 MEM_FILE = dst_file
 445
 446
 447 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 448         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 449         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 450
 451         # First let's fill up these categories
 452         files_new = []
 453         files_removed = []
 454         Dir_removed = []
 455         Dir_Added = []
 456         files_changed = []
 457         files_unchanged = []
 458         files_renamed = []
 459         File_Attibutes = []
 460         Sym_Attibutes = []
 461
 462         files_Del_List = {}
 463         files_New_List = {}
 464         MyDict_Patches = {}
 465
 466         PWD = os.getcwd()
 467
 468         # Generate NEW List
 469         for elt in New_files:
 470                 if elt not in Old_files:
 471                         files_new.append(elt)
 472                         logging.info('New files %s' % elt)
 473
 474         # Generate Delete List
 475         for elt in Old_files:
 476                 if elt not in New_files:
 477                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 478                         files_removed.append(elt)
 479                         logging.info('Old files %s' % elt)
 480
 481         for elt in Old_dirs:
 482                 #print('List of Old Dirs %s' % elt)
 483                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 484                 if elt not in New_dirs:
 485                         Dir_removed.append(elt)
 486                         logging.info('Old Dirs %s' % elt + '/')
 487
 488         for elt in New_dirs:
 489                 if elt not in Old_dirs:
 490                         Dir_Added.append(elt)
 491                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 492
 493         # What files have changed contents but not name/path?
 494         for elt in New_files:
 495                 if elt in Old_files:
 496                         # Both are symbolic linkes and they differ
 497                         src_file = BASE_OLD + '/' + elt
 498                         dst_file = BASE_NEW + '/' + elt
 499                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 500                         if os.path.islink(src_file) and os.path.islink(dst_file):
 501                                 if not os.readlink(src_file) == os.readlink(dst_file):
 502                                         files_changed.append(elt)
 503                                         #print('%d Sym link files changed' % len(files_changed))
 504                                         logging.info('Sym links Changed - %s' % elt)
 505                                 else:
 506                                         files_unchanged.append(elt)
 507                         # Both are Normal files and they differ. (Is file returns true in case of symlink also, so additional check to find either of the file is symlink)
 508                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(src_file) and os.path.isfile(dst_file):
 509                                 if not filecmp.cmp(src_file, dst_file):
 510                                         files_changed.append(elt)
 511                                         #print('%d Normal files changed' % len(files_changed))
 512                                         #print('Files Changed - %s' % elt)
 513                                 else:
 514                                         files_unchanged.append(elt)
 515                         # File types differ between BASE and TARGET
 516                         else:
 517                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 518                                 # Both file types have changed and they differ
 519                                 # Case 1: First Delete the OLD entry file type (Be it anything)
 520                                 # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 521                                 files_removed.append(elt)
 522                                 files_new.append(elt)
 523
 524         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 525         # This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 526
 527         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 528         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 529                 with open(VERBATIM_LIST, 'r') as F_News:
 530                         lines = set(F_News.read().splitlines())
 531                 for line in lines:
 532                         if line in files_changed:
 533                                 files_changed.remove(line)
 534                                 files_removed.append(line)
 535                         if line in files_new:
 536                                 files_new.remove(line)
 537
 538         # Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 539         if SUPPORT_RENAME == "TRUE":
 540                 for elt in files_removed:
 541                         if os.path.isfile(BASE_OLD + '/' + elt):
 542                                 FileName = path_leaf(elt)
 543                                 entries = re.split('[0-9]', FileName)
 544                                 # Gives the STRING part of NAME. if name starts with version then later part wil b string
 545                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 546                                 # If version is starting at the begining of the string?? shd we hav additional check for such cases??
 547                                 if len(entries[0]) > 0:
 548                                         files_Del_List.update({entries[0]: elt})
 549
 550                 for elt in files_new:
 551                         if os.path.isfile(BASE_NEW + '/' + elt):
 552                                 FileName = path_leaf(elt)
 553                                 entries = re.split('[0-9]', FileName)
 554                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 555                                 if len(entries[0]) > 0:
 556                                         files_New_List.update({entries[0]: elt})
 557
 558                 for key, value in files_Del_List.iteritems():
 559                         #print('Key value pair -%s -%s' % (key, value))
 560                         if key in files_New_List:
 561                                 # this file is the same name in both!
 562                                 src_file = BASE_OLD + '/' + value
 563                                 dst_file = BASE_NEW + '/' + files_New_List[key]
 564                                 olddirpath = path_head(files_New_List[key])
 565                                 newdirpath = path_head(value)
 566                                 if os.path.islink(src_file) or os.path.islink(dst_file):
 567                                         logging.debug('Cannot diff as one of them is Symlink')
 568                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 569                                         logging.debug('Cannot diff as one of them is dir')
 570                                 else:
 571                                         #Pick the best diff of same type and diff names
 572                                         files_renamed.append([files_New_List[key], value])
 573                                         files_removed.remove(value)
 574                                         files_new.remove(files_New_List[key])
 575
 576         '''
 577         Patch Section
 578                 Partition.txt contains Protocol for UPI
 579                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 580         '''
 581         Sym_Diff_Cnt = 0
 582         Sym_New_Cnt = 0
 583         Del_Cnt = 0
 584         New_Cnt = 0
 585         Diff_Cnt = 0
 586         Move_Cnt = 0
 587         Verbatim_Cnt = 0
 588         SymLinkDoc = OUT_DIR + '/' + PART_NAME + SYMLINK_DOC_NAME
 589         Partition_Doc = open(OUT_DIR + '/' + PART_NAME + '.txt', 'w')
 590         Partition_Doc_SymLinks = open(SymLinkDoc, 'w')
 591
 592         print("writing diff'ed changed files...")
 593         for elt in files_changed:
 594                 dst_file = BASE_NEW + '/' + elt
 595                 src_file = BASE_OLD + '/' + elt
 596                 # Both files are symbolic links and they differ
 597                 if os.path.islink(dst_file) and os.path.islink(src_file):
 598                         # Both are symlinks and they differ
 599                         logging.debug(' File Changed is Link %s ' % dst_file)
 600                         patch = os.readlink(dst_file)
 601                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 602                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 603                         Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 604                 # Both are NORMAL files and they differ
 605                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) and os.path.isfile(dst_file) and os.path.isfile(src_file):
 606                         # Both are files and they differ
 607                         Diff_Cnt = Diff_Cnt + 1
 608                         patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 609                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 610                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 611                         SS_UpdateSize(src_file, dst_file)
 612
 613                         FORMAT = "REG"
 614                         ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
 615                         if ret is not 0:
 616                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 617                                 files_new.append(elt)
 618                                 Diff_Cnt = Diff_Cnt - 1
 619                         else:
 620                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 621
 622                         Update_Attr(elt, "FILE", File_Attibutes, Sym_Attibutes)
 623                 # Both differ but they are of diff types
 624                 else:
 625                         # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 626                         files_removed.append(elt)
 627                         files_new.append(elt)
 628
 629         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new)
 630         for oldpath, newpath in fdupes.iteritems():
 631                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 632
 633         for elt in files_removed:
 634                 src_file = BASE_OLD + '/' + elt
 635                 # If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 636                 if src_file in fdupes.keys():
 637                         dst_file = BASE_NEW + '/' + fdupes[src_file]
 638                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 639                         Move_Cnt = Move_Cnt + 1
 640                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 641                         files_removed.remove(elt)
 642                         files_new.remove(fdupes[src_file])
 643
 644         # Should be placed after removing duplicates, else they will be filtered here.
 645         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 646         # First Step: Sort & Filter out unwanted files
 647         # Minimum condition used is,
 648         #       1. File name should match 70%
 649         #       2. Extensions should be same
 650         #       3. File name length shd b greater than 3 char
 651         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 652         #       5. Should consider editdistance for RENAME LOGIC ==> TBD
 653
 654         Base_DelList = files_removed[:]
 655         Base_NewList = files_new[:]
 656         DelList = sorted(Base_DelList, key=path_leaf)
 657         NewList = sorted(Base_NewList, key=path_leaf)
 658         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 659
 660         Filter1 = []
 661         Filter2 = []
 662         # Remove unwanted items which we cant make diff with for rename logic
 663         for file in DelList:
 664                 if os.path.islink(BASE_OLD + '/' + file):
 665                         continue
 666                 elif os.path.isdir(BASE_OLD + '/' + file):
 667                         continue
 668                 else:
 669                         Filter1.append(file)
 670                         #logging.debug('Sorted del list - %s' % (file))
 671
 672         DelList = Filter1
 673
 674         for file in NewList:
 675                 if os.path.islink(BASE_NEW + '/' + file):
 676                         continue
 677                 elif os.path.isdir(BASE_NEW + '/' + file):
 678                         continue
 679                 elif len(path_leaf(file)) <= 3:
 680                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW + '/' + file))
 681                         continue
 682                 else:
 683                         Filter2.append(file)
 684
 685         NewList = Filter2
 686
 687         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 688
 689         for new_file in NewList:
 690                 R_Flag = 'FALSE'
 691                 DirPathNew = path_head(new_file)
 692                 FileNameNew = path_leaf(new_file)
 693                 DiffSize = 0
 694                 winning_patch_sz = os.path.getsize(BASE_NEW + '/' + new_file)
 695                 New_fs = winning_patch_sz
 696                 winning_file = ''
 697
 698                 for del_file in DelList:
 699                         FileNameOld = path_leaf(del_file)
 700                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 701                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 702                                 # Percentage difference between two file sizes is within 30%, then we consider for diff generation
 703                                 Del_fs = os.path.getsize(BASE_OLD + '/' + del_file)
 704                                 v1 = abs(New_fs - Del_fs)
 705                                 v2 = (New_fs + Del_fs) / 2
 706                                 if(v2 <= 0 or ((v1 / v2) * 100) > 30):
 707                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 708                                         continue
 709                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 710                                 R_Flag = 'TRUE'
 711                                 DiffSize = measure_two_filediffs(BASE_OLD + '/' + del_file, BASE_NEW + '/' + new_file)
 712                                 if (DiffSize < 0.8 * winning_patch_sz):
 713                                         winning_patch_sz = DiffSize
 714                                         winning_file = del_file
 715                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and R_Flag == 'TRUE'):
 716                                 logging.debug('Becuase nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 717                                 break
 718                 if len(winning_file) > 0:
 719                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 720                         files_renamed.append([new_file, winning_file])
 721                         DelList.remove(winning_file)
 722                         files_removed.remove(winning_file)
 723                         files_new.remove(new_file)
 724
 725         #********************** Files should NOT be deleted for any such renames ***********************
 726
 727         if SUPPORT_RENAME == "TRUE":
 728                 for elt in files_renamed:
 729                         src_file = BASE_OLD + '/' + elt[1]
 730                         dst_file = BASE_NEW + '/' + elt[0]
 731                         Diff_Cnt = Diff_Cnt + 1
 732                         patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 733                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 734                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 735                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 736                         # Should be careful of renaming files??
 737                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 738                         # What if folder path has numerics??
 739
 740                         if os.path.isdir(src_file) or os.path.isdir(dst_file):
 741                                 # This case never occurs??
 742                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 743                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 744                         else:  # Make sure these files are PROPER and they shd NOT be symlinks
 745                                 if filecmp.cmp(src_file, dst_file):
 746                                         Move_Cnt = Move_Cnt + 1
 747                                         Diff_Cnt = Diff_Cnt - 1
 748                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 749                                 else:
 750                                         FORMAT = "REG"
 751                                         ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
 752                                         if ret is not 0:
 753                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 754                                                 files_new.append(elt)
 755                                                 Diff_Cnt = Diff_Cnt - 1
 756                                         else:
 757                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 758
 759                                 SS_UpdateSize(src_file, dst_file)
 760                                 Update_Attr(elt[0], "FILE", File_Attibutes, Sym_Attibutes)
 761
 762         # HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 763         # So that, the rename functionality will NOT create PATCH instead of verbatims.
 764
 765         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 766                 with open(VERBATIM_LIST, 'r') as F_News:
 767                         lines = set(F_News.read().splitlines())
 768                 for line in lines:
 769                         if line not in files_new:
 770                                 if os.path.exists(BASE_NEW + '/' + line):
 771                                         files_new.append(line)
 772                                         Verbatim_Cnt = Verbatim_Cnt + 1
 773                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW + '/' + line)
 774
 775         for elt in files_removed:
 776                 # if files are part of patches after renaming, we shd remove them as part of removed.
 777                 src_file = BASE_OLD + '/' + elt
 778                 if os.path.islink(src_file):
 779                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 780                 elif os.path.isdir(src_file):
 781                         # If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 782                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 783                 else:
 784                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 785                 logging.debug(' File Deleted %s' % src_file)
 786                 Del_Cnt = Del_Cnt + 1
 787
 788         Dir_removed.sort(reverse=True)
 789         for elt in Dir_removed:
 790                 # if Dir is empty, add it to the removed list.
 791                 src_file = BASE_OLD + '/' + elt
 792                 # Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 793                 # It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 794                 if os.path.isdir(src_file):
 795                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 796                         Del_Cnt = Del_Cnt + 1
 797                         logging.debug(' Dir Deleted- %s' % src_file)
 798
 799         for elt in files_new:
 800                 dst_file = BASE_NEW + '/' + elt
 801                 newfiles_dest_path = 'run/upgrade-sysroot/'
 802                 try:
 803                         ensure_dir_exists(newfiles_dest_path)
 804                 except FileExistsError as exc:
 805                         logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path)
 806                         raise exc
 807                 if os.path.islink(dst_file):
 808                         patch = os.readlink(dst_file)
 809                         logging.debug(' File New Links %s' % elt)
 810                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 811                         # What if this is only a new sym link and folder already exists??? Should recheck
 812                         destpath = newfiles_dest_path + elt
 813                         if not os.path.exists(path_head(destpath)):
 814                                 os.makedirs(path_head(destpath))
 815                                 logging.info('New SymLink - Adding missing Dir')
 816                         #Update_Attr(elt, "SYM", File_Attibutes, Sym_Attibutes)
 817                         Sym_New_Cnt = Sym_New_Cnt + 1
 818                 elif os.path.isdir(dst_file):  # We create just empty directory here
 819                         destpath = newfiles_dest_path + elt
 820                         if not os.path.exists(destpath):
 821                                 os.makedirs(destpath)
 822                                 logging.debug(' File New Dir %s' % destpath)
 823                                 New_Cnt = New_Cnt + 1
 824                 else:
 825                         New_Cnt = New_Cnt + 1
 826                         destpath = newfiles_dest_path + elt
 827                         destdir = os.path.dirname(destpath)
 828                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 829
 830                         if not os.path.isdir(destdir):
 831                                 try:
 832                                         os.makedirs(destdir)
 833                                 except Exception as exc:
 834                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 835                                         raise exc
 836
 837                         try:
 838                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 839                                         shutil.copy2(dst_file, destpath)
 840                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 841                         except Exception as exc:
 842                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 843                                 raise exc
 844
 845         for elt in Dir_Added:
 846                 newfiles_dest_path = 'run/upgrade-sysroot/'
 847                 try:
 848                         ensure_dir_exists(newfiles_dest_path)
 849                 except FileExistsError as exc:
 850                         logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path)
 851                         raise exc
 852                 destpath = newfiles_dest_path + elt
 853                 if not os.path.exists(destpath):
 854                         os.makedirs(destpath)
 855                         logging.debug(' DirList New Dir %s' % destpath)
 856                         New_Cnt = New_Cnt + 1
 857
 858         # Base directory should be system
 859         print 'Compressing New files'
 860         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 861                 WorkingDir = os.getcwd()
 862                 os.chdir(os.getcwd() + "/" + NEW_FILES_PATH)
 863                 logging.info('Curr Working Dir - %s' % os.getcwd())
 864                 os.system(ZIPUTIL + NEW_FILES_PATH + " >> " + LOGFILE)
 865                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR)
 866                 # New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 867                 SS_UpdateSize(WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME)
 868                 os.chdir(WorkingDir)
 869                 shutil.rmtree(NEW_FILES_PATH)
 870                 # use 7z a system.7z ./*
 871
 872         #logging.info('%d Dir to be removed' % len(Dir_removed))
 873         logging.info('%d files unchanged' % len(files_unchanged))
 874         logging.info('%d files files_renamed' % len(files_renamed))
 875         logging.info('%d files NEW' % len(files_new))
 876         logging.info('%d File attr' % len(File_Attibutes))
 877         logging.info('%d Sym attr' % len(Sym_Attibutes))
 878         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 879         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d Verbatim -%d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Verbatim_Cnt))
 880
 881         # There could be duplicates, TODO, can check before adding..
 882         ATTR_FILE_D = open(ATTR_FILE, 'a+')
 883         for elt in File_Attibutes:
 884                 ATTR_FILE_D.write(elt)
 885         for elt in Sym_Attibutes:
 886                 ATTR_FILE_D.write(elt)
 887
 888         ATTR_FILE_D.close()
 889
 890         Partition_Doc_SymLinks.close()
 891         Partition_Read_SymLinks = open(SymLinkDoc, 'r+')
 892         Partition_Doc.write(Partition_Read_SymLinks.read())
 893         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d\n' % (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt))
 894         Partition_Doc_SymLinks.close()
 895         Partition_Doc.close()
 896         os.remove(SymLinkDoc)
 897
 898         if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + os.path.getsize(ATTR_FILE) == 0:
 899                 print('No Delta Generated for %s - %s' % (PART_NAME, OUT_DIR))
 900                 logging.info('No Delta Generated for %s' % PART_NAME)
 901                 shutil.rmtree(OUT_DIR)
 902
 903
 904 def Apply_Container_Delta(a_apk, b_apk, new_apk, a_folder, g_output_dir):
 905
 906         # CONTROL NAMES, AND PRINTS AND ERROR CASES... SHOULD NOT PROCEED.
 907         print 'ApplyContainerDelta - ', b_apk, a_folder, g_output_dir
 908         shutil.copy2(g_output_dir + '/' + b_apk, g_output_dir + '/temp')
 909         temp_apk = '../' + g_output_dir + '/' + b_apk
 910         Patch = 'Patch_' + b_apk
 911         ensure_dir_exists(Patch)
 912         shutil.copy2(g_output_dir + '/' + b_apk, Patch + '/' + b_apk)
 913
 914         # Size issue on Device side?? shd check this
 915         subprocess.call(['unzip', '-q', Patch + '/' + b_apk, '-d', Patch])
 916         with open(g_output_dir + '/PATCH.txt', 'r') as f_new:
 917                 lines = set(f_new.read().splitlines())
 918                 for line in lines:
 919                         #print('Action ==> %s' % line)
 920                         #Action, Path, Patch = line.split('|')
 921                         Items = line.split('|')
 922                         Action = Items[0]
 923                         Path = Items[1]
 924                         ActualPath = a_folder + '/' + Path
 925                         PatchPath = Patch + '/' + Path
 926                         SrcPath = g_output_dir + '/' + path_leaf(Path)
 927                         #print('Action ==> %s Path ==> %s ' % (Action, Path))
 928                         if line[0] == 'c':
 929                                 patchName = g_output_dir + '/' + Items[2]
 930                                 #print('Apply Patch: ActualPath %s SrcPath %s PatchLoc %s ' % (PatchPath, ActualPath, patchName))
 931                                 subprocess.call([DIFFPATCH_UTIL, ActualPath, ActualPath, patchName])
 932                                 WorkingDir = os.getcwd()
 933                                 os.chdir(WorkingDir + "/" + "temp_a")
 934                                 subprocess.call(['cp', '--parents', Path, '../' + Patch])
 935                                 os.chdir(WorkingDir)
 936                         elif line[0] == 's':
 937                                 WorkingDir = os.getcwd()
 938                                 os.chdir(WorkingDir + "/" + "temp_a")
 939                                 subprocess.call(['cp', '--parents', Path, '../' + Patch])
 940                                 os.chdir(WorkingDir)
 941                         else:
 942                                 print('Apply_Container_Delta - Unknown Error')
 943         #print('Touch all files and set common attributes for DIFF generation')
 944         WorkingDir = os.getcwd()
 945         os.chdir(WorkingDir + "/" + Patch)
 946
 947         CONTAINER_DATE = '200011111111.11'
 948         CONTAINER_MODE = '0755'
 949         subprocess.call(['find', '.', '-type', 'l', '-exec', 'rm', '-rf', '{}', ';'])
 950         subprocess.call(['find', '.', '-exec', 'touch', '-t', CONTAINER_DATE, '{}', ';'])
 951         subprocess.call(['chmod', '-R', CONTAINER_MODE, '../' + Patch])
 952
 953         print 'Update Intermediate Archive'
 954         #subprocess.call(['zip','-ryX', b_apk, '*'])
 955         subprocess.call(['zip', '-ryX', b_apk] + glob.glob('*'))
 956         os.chdir(WorkingDir)
 957         #print('Apply Path completed - Now create diff for this and place in patch folder')
 958         #print os.getcwd()
 959         print('Patch Applied, Create Final Diff - %s %s' % (g_output_dir + '/' + b_apk, new_apk))
 960         patchName = ('New' + '_%s' + DIFF_SUFFIX) % (b_apk)
 961         patchLoc = '%s/%s' % (g_output_dir, patchName)
 962
 963         subprocess.call([DIFF_UTIL, Patch + '/' + b_apk, new_apk, patchLoc])
 964
 965         # Only on HOST... for testing
 966         if TEST_MODE == 'TRUE':
 967                 UpgradedName = '%s_Upgraded' % (b_apk)
 968                 subprocess.call([DIFFPATCH_UTIL, Patch + '/' + b_apk, UpgradedName, patchLoc])
 969
 970         # This is file only with NEWS and empty diffs and same files.
 971         if TEST_MODE == 'FALSE':
 972                 os.remove(g_output_dir + '/' + b_apk)
 973                 os.rename(g_output_dir + '/temp', g_output_dir + '/' + b_apk)
 974                 shutil.rmtree(Patch)
 975
 976
 977 def IsSymlink(info):
 978         return (info.external_attr >> 16) == 0120777
 979
 980
 981 def NewFiles(src, dest):
 982         print src, dest
 983         subprocess.call(['cp', '-rp', src, dest])
 984         #try:
 985                 #shutil.copytree(src, dest)
 986         #except OSError as e:
 987                 # If the error was caused because the source wasn't a directory
 988                 #if e.errno == errno.ENOTDIR:
 989                         #shutil.copy2(src, dest)
 990                 #else:
 991                         #print('Directory not copied. Error: %s' % e)
 992
 993
 994 def measure_two_filediffs(src, dst):
 995         patchLoc = 'temp.patch'
 996         subprocess.call([DIFF_UTIL, src, dst, patchLoc])
 997         result_size = os.path.getsize(patchLoc)
 998         os.remove(patchLoc)
 999         return result_size
1000
1001
1002 def Get_Files(path):
1003         all_files = []
1004         all_dirs = []
1005
1006         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
1007                 for directory in directories:
1008                         #DirName = os.path.join(root+'/',directory)
1009                         DirName = os.path.join(root, directory)
1010                         if os.path.islink(DirName):
1011                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
1012                                 all_files.append(os.path.relpath(DirName, path))
1013                         elif not os.listdir(DirName):
1014                                 #print('*****Empty Directory******* -%s', DirName)
1015                                 # This should NOT be appended ??? Empty dir shd b considered
1016                                 all_dirs.append(os.path.relpath(DirName, path))
1017                         else:
1018                                 all_dirs.append(os.path.relpath(DirName, path))
1019                 for filename in filenames:
1020                         FileName = os.path.join(root, filename)
1021                         all_files.append(os.path.relpath(FileName, path))
1022
1023         all_files.sort()
1024         all_dirs.sort()
1025         return all_files, all_dirs
1026
1027
1028 USAGE_DOCSTRING = """
1029         Generate Delta using BASEOLD AND BASE NEW
1030         Attributes is optional
1031         Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
1032 """
1033
1034
1035 def Usage(docstring):
1036         print docstring.rstrip("\n")
1037         print COMMON_DOCSTRING
1038
1039
1040 if __name__ == '__main__':
1041         main()