mk_delta/common/bin/CreatePatch.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import os
   5 import filecmp
   6 import shutil
   7 import subprocess
   8 import re
   9 import ntpath
  10 import zipfile
  11 import datetime
  12 import hashlib
  13 import operator
  14 import locale
  15 import errno
  16 import logging
  17 import glob
  18 import apt
  19 import stat
  20
  21 if sys.hexversion < 0x02040000:
  22         print >> sys.stderr, "Python 2.4 or newer is required."
  23         sys.exit(1)
  24
  25 '''
  26 Diff two folders and create delta using SS_BSDIFF
  27 Will maintain same format of script that will be generated when we use diffutil
  28
  29 1. Create a list of files in each Base folders,
  30 2. These files will fall into one these below categories:
  31         1) Only in OLD - Should be deleted
  32         2) Only in NEW - Should be added or renamed accordingly
  33         3) File exists in both directories but contents are different - Create Diff.
  34         4) File name is same but TYPE can change (File to Folder, Folder to Link etc.)
  35         5) Duplicates in the list of Deletes and News
  36         6) Close matching diffs even though name changes across directories. (for matching extension)
  37         7) Clearing empty directories after Moves or diffs under Rename.
  38         8) Supporting Verbatim - Any entry under Verbatim_list.txt will be treated as NEW files instead of patch.
  39
  40 Current Case
  41 1. Given two folders, from list of REMOVED and NEW files find if there
  42 is version change and create diff between them
  43
  44 TODO
  45 Want to extend the same script for entire DIFF generation and replace TOTAlib.sh file
  46 Catching errors at all stages. SHOULD exit & return error in case of failure
  47 '''
  48
  49
  50 def global_paths():
  51         global DIFF_UTIL
  52         global ZIPUTIL
  53         global NEW_FILES_PATH
  54         global NEW_FILES_ZIP_NAME
  55         global SYMLINK_TYPE
  56         global ATTR_DOC_EXT
  57         global SYMLINK_DOC_NAME
  58         global DIFF_PREFIX
  59         global DIFF_SUFFIX
  60         global SUPPORT_RENAME
  61         global NEW_PREFIX
  62         global DIFFPATCH_UTIL
  63         global SUPPORT_CONTAINERS
  64         global FULL_IMAGE
  65         global DELTA_IMAGE
  66         global DELTA_FS
  67         global EXTRA
  68         global COMMON_BIN_PATH
  69         global MEM_REQ
  70         global EMPTY
  71         global VERBATIM_LIST
  72         global MEM_FILE
  73
  74
  75 COMMON_BIN_PATH = "../../common/bin/"
  76 DIFF_UTIL = "/usr/local/bin/ss_bsdiff"
  77 DIFFPATCH_UTIL = "/usr/local/bin/ss_bspatch"
  78 #ZIPUTIL = "p7zip "
  79 ZIPUTIL = "7z -mf=off a system.7z "
  80 NEW_FILES_PATH = "run/upgrade-sysroot"
  81 NEW_FILES_ZIP_NAME = "system.7z"
  82 SYMLINK_TYPE = "SYM"
  83 ATTR_DOC_EXT = "_attr.txt"
  84 SYMLINK_DOC_NAME = "_sym.txt"
  85 HARDLINK_DOC_NAME = "_hard.txt"
  86 PART_DOC_EXT = ".txt"
  87 DIFF_PREFIX = "diff"
  88 DIFF_SUFFIX = ".delta"
  89 NEW_PREFIX = 'new'
  90 FULL_IMAGE = "FULL_IMAGE"
  91 DELTA_IMAGE = "DELTA_IMAGE"
  92 DELTA_FS = "DELTA_FS"
  93 EXTRA = "EXTRA"
  94 LOGFILE = "Delta.log"
  95 VERBATIM_LIST = "Verbatim_List.txt"
  96 EMPTY = ""
  97 MEM_REQ = 0
  98 MEM_FILE = "NULL"
  99 COMPRESSION_LZMA = "lzma"
 100 COMPRESSION_BROTLI = "brotli"
 101
 102 SUPPORT_RENAME = "TRUE"  # Use appropriate name
 103 SUPPORT_CONTAINERS = "FALSE"
 104 SUPPORT_VERBATIM = "TRUE"
 105
 106 TEST_MODE = "FALSE"
 107
 108
 109 def main():
 110         logging.basicConfig(filename=LOGFILE, level=logging.DEBUG)
 111         global AttributeFile
 112         global GenerateDiffAttr
 113         try:
 114
 115                 if len(sys.argv) < 5:
 116                         sys.exit('Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER')
 117                 UPDATE_TYPE = sys.argv[1]
 118                 UPDATE_TYPE_S = UPDATE_TYPE.split(":")
 119                 PART_NAME = sys.argv[2]  # lets make this also optional
 120
 121                 BASE_OLD = sys.argv[3]
 122                 BASE_NEW = sys.argv[4]
 123                 OUT_DIR = sys.argv[5]
 124                 ATTR_OLD = EMPTY
 125                 ATTR_NEW = EMPTY
 126                 UPDATE_CFG_PATH = EMPTY
 127                 GenerateDiffAttr = "FALSE"
 128                 if UPDATE_TYPE_S[0] == DELTA_FS:
 129                         #instead of arguments check it in outdirectory ?
 130                         if len(sys.argv) == 9:
 131                                 ATTR_OLD = sys.argv[6]
 132                                 ATTR_NEW = sys.argv[7]
 133                                 UPDATE_CFG_PATH = '../' + sys.argv[8]
 134                                 GenerateDiffAttr = "TRUE"
 135
 136                 elif UPDATE_TYPE_S[0] in [DELTA_IMAGE, FULL_IMAGE]:
 137                         if len(sys.argv) == 7:
 138                                 #Use path in better way
 139                                 UPDATE_CFG_PATH = '../' + sys.argv[6]
 140
 141                 global DIFF_UTIL
 142                 global DIFFPATCH_UTIL
 143                 if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 144                         DIFF_UTIL = COMMON_BIN_PATH + DIFF_UTIL
 145                         DIFFPATCH_UTIL = COMMON_BIN_PATH + DIFFPATCH_UTIL
 146                         if not (os.path.isfile(DIFF_UTIL) and os.access(DIFF_UTIL, os.X_OK)):
 147                                 print >> sys.stderr, "Diff Util Does NOT exist -- ABORT"
 148                                 logging.info('Diff Util Does NOT exist -- ABORT')
 149                                 sys.exit(1)
 150
 151                 start = datetime.datetime.now().time()
 152                 logging.info('*************** ENTERED PYTHON SCRIPT *****************')
 153                 logging.info('Arguments Passed: [UpdateType - %s][Part Name - %s] [BaseOld - %s]  [BaseNew - %s] \n [OUTPUTDir - %s] [BASE ATTR - %s] [TARGET ATTR - %s]' % (UPDATE_TYPE, PART_NAME, BASE_OLD, BASE_NEW, OUT_DIR, ATTR_OLD, ATTR_NEW))
 154
 155                 try:
 156                         ensure_dir_exists(OUT_DIR)
 157                 except FileExistsError as exc:
 158                         logging.error('Argument passed as OUT_DIR - %s is already an existing file' % OUT_DIR)
 159                         raise exc
 160                 if GenerateDiffAttr == "TRUE":
 161                         if not (os.path.isfile(ATTR_OLD) and os.path.isfile(ATTR_NEW)):
 162                                 print >> sys.stderr, "Attributes missing -- ABORT"
 163                                 sys.exit(1)
 164
 165                 # Should check if APT is supported on other linux flavours
 166                 cache = apt.Cache()
 167                 if cache['p7zip'].is_installed and cache['attr'].is_installed and cache['tar'].is_installed:
 168                         logging.info('Basic utils installed')
 169                 else:
 170                         print >> sys.stderr, "Basic utils missing -- ABORT"
 171                         sys.exit(1)
 172
 173                 if UPDATE_TYPE_S[0] == FULL_IMAGE:
 174                         SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH)
 175                 # #### currently does not support LZMA ####
 176                 #  elif UPDATE_TYPE == DELTA_IMAGE:
 177                 #      SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_LZMA)
 178                 elif UPDATE_TYPE_S[0] == DELTA_IMAGE:
 179                         SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, PART_NAME, UPDATE_CFG_PATH, COMPRESSION_BROTLI)
 180                 elif UPDATE_TYPE == DELTA_FS:
 181                         AttributeFile = ATTR_NEW
 182                         ATTR_FILE = OUT_DIR + '/' + PART_NAME + ATTR_DOC_EXT
 183                         Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE)
 184                         Old_files, Old_dirs = Get_Files(BASE_OLD)
 185                         New_files, New_dirs = Get_Files(BASE_NEW)
 186                         SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE)
 187
 188                         if not UPDATE_CFG_PATH == EMPTY:
 189                                 SS_update_cfg(PART_NAME, UPDATE_CFG_PATH)
 190
 191                 elif UPDATE_TYPE == EXTRA:
 192                         print('UPDATE_TYPE ---- EXTRA')
 193                 else:
 194                         print('UPDATE_TYPE ---- UNKNOWN FORMAT')
 195
 196                 if GenerateDiffAttr == "TRUE":
 197                         if os.path.exists(ATTR_OLD) and os.path.exists(ATTR_NEW):
 198                                 os.remove(ATTR_OLD)
 199                                 os.remove(ATTR_NEW)
 200                 end = datetime.datetime.now().time()
 201
 202                 logging.info('Max Memory requried to upgrade [%s] is [%d] for File[%s]' % (PART_NAME, MEM_REQ, MEM_FILE))
 203                 logging.info('*************** DONE WITH PYTHON SCRIPT ***************')
 204                 logging.info('Time start [%s] - Time end [%s]' % (start, end))
 205                 print('Done with [%s][%d]---- Time start [%s] - Time end [%s]' % (PART_NAME, MEM_REQ, start, end))
 206
 207         except Exception as exc:
 208                 logging.error('Usage: {} <Update_Type> <Part_Name> <OLD_Base> <NEW_Base> <OUT_DIR>'.format(os.path.basename(sys.argv[0])))
 209                 raise exc
 210
 211
 212 def SS_update_cfg(DELTA_BIN, UPDATE_CFG_PATH):
 213         f = open(UPDATE_CFG_PATH, 'r')
 214         lines = f.readlines()
 215         f.close()
 216         f = open(UPDATE_CFG_PATH, 'w')
 217         for line in lines:
 218                 ConfigItems = line.split()
 219                 if ConfigItems[0] == DELTA_BIN:
 220                         DELTA = ConfigItems[1]
 221                         logging.info('Updating %s config' % DELTA_BIN)
 222                         line = line.rstrip('\n')
 223                         Value = MEM_REQ
 224                         line = line.replace(line, line + '\t' + str(Value) + '\n')
 225                         f.write(line)
 226                 else:
 227                         f.write(line)
 228         f.close()
 229
 230
 231 def SS_mk_delta_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH, COMPRESSION_METHOD):
 232         #for sizes
 233
 234         oldsize_d = os.path.getsize(BASE_OLD)
 235         newsize_d = os.path.getsize(BASE_NEW)
 236         SHA_BIN_DEST = hash_file(BASE_NEW)
 237         SHA_BIN_BASE = hash_file(BASE_OLD)
 238
 239         #incase UPDATE CFG is empty
 240         DELTA = DELTA_BIN
 241         SS_UpdateSize(BASE_OLD, BASE_NEW)
 242         #Should throw error if PART NAME NOT found??
 243         if not UPDATE_CFG_PATH == EMPTY:
 244                 f = open(UPDATE_CFG_PATH, 'r')
 245                 lines = f.readlines()
 246                 f.close()
 247                 f = open(UPDATE_CFG_PATH, 'w')
 248                 for line in lines:
 249                         ConfigItems = line.split()
 250                         if ConfigItems[0] == DELTA_BIN:
 251                                 logging.info('Updating %s config' % DELTA_BIN)
 252                                 DELTA = ConfigItems[1]
 253                                 line = line.rstrip('\n')
 254                                 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
 255                                 f.write(line)
 256                         else:
 257                                 f.write(line)
 258                 f.close()
 259
 260         patchLoc = '%s/%s' % (OUT_DIR, DELTA)
 261         logging.info('Make Delta Image %s <--> %s ==> %s %s' % (BASE_OLD, BASE_NEW, DELTA_BIN, patchLoc))
 262         subprocess.call([DIFF_UTIL, "-c", COMPRESSION_METHOD, BASE_OLD, BASE_NEW, patchLoc])
 263
 264
 265 def SS_mk_full_img(BASE_OLD, BASE_NEW, OUT_DIR, DELTA_BIN, UPDATE_CFG_PATH):
 266         logging.info('Make Full Image %s <--> %s ==> %s' % (BASE_OLD, BASE_NEW, DELTA_BIN))
 267         oldsize_d = os.path.getsize(BASE_OLD)
 268         newsize_d = os.path.getsize(BASE_NEW)
 269         SHA_BIN_DEST = hash_file(BASE_NEW)
 270         SHA_BIN_BASE = hash_file(BASE_OLD)
 271         #echo -e "\t${oldsize_d}\t\t${newsize_d}\t\t${SHA_BIN_BASE}\t\t${SHA_BIN_DEST}" >> ${DATA_DIR}/update_new.cfg
 272         SS_UpdateSize(BASE_OLD, BASE_NEW)
 273
 274         if not UPDATE_CFG_PATH == EMPTY:
 275                 f = open(UPDATE_CFG_PATH, 'r')
 276                 lines = f.readlines()
 277                 f.close()
 278                 f = open(UPDATE_CFG_PATH, 'w')
 279                 for line in lines:
 280                         ConfigItems = line.split()
 281                         if ConfigItems[0] == DELTA_BIN:
 282                                 logging.info('Updating %s config' % DELTA_BIN)
 283                                 DELTA = ConfigItems[1]
 284                                 line = line.rstrip('\n')
 285                                 line = line.replace(line, line + '\t' + str(oldsize_d) + '\t\t' + str(newsize_d) + '\t\t' + str(SHA_BIN_BASE) + '\t\t' + str(SHA_BIN_DEST) + '\n')
 286                                 f.write(line)
 287                         else:
 288                                 f.write(line)
 289                 f.close()
 290
 291
 292 def zipdir(path, zip):
 293         for root, dirs, files in os.walk(path):
 294                 for file in files:
 295                         zip.write(os.path.join(root, file))
 296
 297
 298 def ensure_dir_exists(path):
 299         if not os.path.exists(path):
 300                 os.makedirs(path)
 301         elif os.path.isfile(path):
 302                 raise FileExistsError
 303                 #shutil.rmtree(path)
 304         #os.makedirs(path)
 305
 306
 307 def path_leaf(path):
 308         head, tail = ntpath.split(path)  # This is for windows?? Recheck
 309         return tail
 310
 311
 312 def path_head(path):
 313         head, tail = ntpath.split(path)
 314         return head
 315
 316
 317 # Creating Diff between OLD and NEW attribute files v12
 318 def Diff_AttrFiles(ATTR_OLD, ATTR_NEW, ATTR_FILE):
 319         if GenerateDiffAttr == "FALSE":
 320                 return
 321         with open(ATTR_OLD, 'r') as f_old:
 322                 lines1 = set(f_old.read().splitlines())
 323
 324         with open(ATTR_NEW, 'r') as f_new:
 325                 lines2 = set(f_new.read().splitlines())
 326
 327         lines = set.difference(lines2, lines1)
 328         with open(ATTR_FILE, 'w+') as file_out:
 329                 for line in lines:
 330                         logging.info('Diff_AttrFiles - %s' % line)
 331                         file_out.write(line + '\n')
 332
 333
 334 def Update_Attr(RequestedPath, Type, File_Attributes, Sym_Attributes):
 335         # Full File Path should MATCH
 336         if GenerateDiffAttr == "FALSE":
 337                 return
 338         FilePath = '"/' + RequestedPath + '"'
 339         #print ('FilePath - %s'% (FilePath))
 340         with open(AttributeFile) as f:
 341                 for line in f:
 342                         if FilePath in line:
 343                                 if Type == SYMLINK_TYPE:
 344                                         Sym_Attributes.append(line)
 345                                 else:
 346                                         File_Attributes.append(line)
 347
 348
 349 def hash_file(filename):
 350         '''This function returns the SHA-1 hash of the file passed into it'''
 351
 352         # make a hash object
 353         h = hashlib.sha1()
 354
 355         # open file for reading in binary mode
 356         with open(filename, 'rb') as file:
 357                 # loop till the end of the file
 358                 chunk = 0
 359                 while chunk != b'':
 360                         # read only 1024 bytes at a time
 361                         chunk = file.read(1024 * 1024)
 362                         h.update(chunk)
 363
 364         # return the hex representation of digest
 365         return h.hexdigest()
 366
 367
 368 def find_dupes_list(BASE_OLD, BASE_NEW, fileListB, fileListT, Old_hardlinks, New_hardlinks):
 369         dups = {}
 370         fdupes = {}
 371         print('Finding Duplicates in - %s %s' % (BASE_OLD, BASE_NEW))
 372
 373         for filename in fileListB:
 374                 Src_File = BASE_OLD + '/' + filename
 375                 if os.path.islink(Src_File) or os.path.isdir(Src_File) or ishardlink(Src_File):
 376                         continue
 377                 # Calculate hash
 378                 file_hash = hash_file(Src_File)
 379                 dups[file_hash] = Src_File
 380
 381         for filename in fileListT:
 382                 Dest_File = BASE_NEW + '/' + filename
 383                 if os.path.islink(Dest_File) or os.path.isdir(Dest_File) or ishardlink(Dest_File):
 384                         continue
 385                 # Calculate hash
 386                 file_hash = hash_file(Dest_File)
 387                 if file_hash in dups:
 388                         BaseStr = dups.get(file_hash)
 389                         Baseloc = BaseStr.find('/')
 390                         if not BaseStr[Baseloc:] == filename:
 391                                 #print('Dupes - %s ==> %s' % (BaseStr[Baseloc:], filename))
 392                                 fdupes[BaseStr] = filename
 393         logging.info('Total Duplicate files %d' % (len(fdupes)))
 394         return fdupes
 395
 396
 397 def SS_UpdateSize(src_file, dst_file):
 398         global MEM_REQ
 399         global MEM_FILE
 400         oldsize_d = os.path.getsize(src_file)
 401         newsize_d = os.path.getsize(dst_file)
 402         if oldsize_d >= newsize_d:
 403                 Max = newsize_d
 404         else:
 405                 Max = oldsize_d
 406         if MEM_REQ < Max:
 407                 MEM_REQ = Max
 408                 MEM_FILE = dst_file
 409
 410
 411 def SS_Generate_Delta(PART_NAME, BASE_OLD, Old_files, Old_dirs, BASE_NEW, New_files, New_dirs, OUT_DIR, ATTR_FILE):
 412         print('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 413         logging.info('Going from %d files to %d files' % (len(Old_files), len(New_files)))
 414
 415         # First let's fill up these categories
 416         files_new = []
 417         files_removed = []
 418         Dir_removed = []
 419         Dir_Added = []
 420         files_changed = []
 421         files_unchanged = []
 422         files_renamed = []
 423         File_Attributes = []
 424         Sym_Attributes = []
 425
 426         files_Del_List = {}
 427         files_New_List = {}
 428
 429         # Get dictionaries used for hardlinks form both directories
 430         New_hardlinks = get_hardlinks(BASE_NEW)
 431         Old_hardlinks = get_hardlinks(BASE_OLD)
 432
 433         # Generate NEW List
 434         for elt in New_files:
 435                 if elt not in Old_files:
 436                         files_new.append(elt)
 437                         logging.info('New files %s' % elt)
 438
 439         # Generate Delete List
 440         for elt in Old_files:
 441                 if elt not in New_files:
 442                         # Cant we just append it here only if this is NOT a directory???? so that we have list of removed files ONLY. including directories
 443                         files_removed.append(elt)
 444                         logging.info('Old files %s' % elt)
 445
 446         for elt in Old_dirs:
 447                 #print('List of Old Dirs %s' % elt)
 448                 # Delete END logic goes in hand with UPG, After Diffs and moves, DEL END should be done.
 449                 if elt not in New_dirs:
 450                         Dir_removed.append(elt)
 451                         logging.info('Old Dirs %s' % elt + '/')
 452
 453         for elt in New_dirs:
 454                 if elt not in Old_dirs:
 455                         Dir_Added.append(elt)
 456                 #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 457
 458         # What files have changed contents but not name/path?
 459         for elt in New_files:
 460                 if elt in Old_files:
 461                         # Both are symbolic linkes and they differ
 462                         src_file = BASE_OLD + '/' + elt
 463                         dst_file = BASE_NEW + '/' + elt
 464                         #print('Files Changed - %s -%s' % (src_file,dst_file))
 465                         if os.path.islink(src_file) and os.path.islink(dst_file):
 466                                 if not (os.readlink(src_file) == os.readlink(dst_file)):
 467                                         files_changed.append(elt)
 468                                         #print('%d Sym link files changed' % len(files_changed))
 469                                         logging.info('Sym links Changed - %s' % elt)
 470                                 else:
 471                                         files_unchanged.append(elt)
 472                         # Both are hardlinks - we add them because we can't be sure if file they point to changes
 473                         elif elt in New_hardlinks and elt in Old_hardlinks:
 474                                 files_changed.append(elt)
 475                         # Both are Normal files and they differ. (Is file returns true in case of sym/hardlink also,
 476                         # so additional check to find either of the file is sym/hardlink)
 477                         elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) \
 478                                 and (not (elt in New_hardlinks or elt in Old_hardlinks)) \
 479                                 and os.path.isfile(src_file) and os.path.isfile(dst_file):
 480                                 if not filecmp.cmp(src_file, dst_file):
 481                                         files_changed.append(elt)
 482                                         #print('%d Normal files changed' % len(files_changed))
 483                                         #print('Files Changed - %s' % elt)
 484                                 else:
 485                                         files_unchanged.append(elt)
 486                         # File types differ between BASE and TARGET
 487                         else:
 488                                 logging.info('Files are of diff types but same names  Src- %s Des- %s' % (src_file, dst_file))
 489                                 # Both file types have changed and they differ
 490                                 # Case 1: First Delete the OLD entry file type (Be it anything)
 491                                 # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 492                                 files_removed.append(elt)
 493                                 files_new.append(elt)
 494
 495         # HANDLING VERBATIM - Remove from changed list and delete the entries on device first
 496         # This script is called partition wise, So, how do u want to handle it? (specialy for delete case?)
 497
 498         print("Check for any verbatim under - %s" % VERBATIM_LIST)
 499         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 500                 with open(VERBATIM_LIST, 'r') as F_News:
 501                         lines = set(F_News.read().splitlines())
 502                 for line in lines:
 503                         if line in files_changed:
 504                                 files_changed.remove(line)
 505                                 files_removed.append(line)
 506                         if line in files_new:
 507                                 files_new.remove(line)
 508
 509         # Currently if Version or number is the first character of the file, then we are NOT making any diffs.
 510         if SUPPORT_RENAME == "TRUE":
 511                 for elt in files_removed:
 512                         if os.path.isfile(BASE_OLD + '/' + elt):
 513                                 FileName = path_leaf(elt)
 514                                 entries = re.split('[0-9]', FileName)
 515                                 # Gives the STRING part of NAME. if name starts with version then later part wil b string
 516                                 #print('Entires under removed list after split - %s %s - %s' % (FileName, entries[0], elt))
 517                                 # If version is starting at the begining of the string?? shd we hav additional check for such cases??
 518                                 if len(entries[0]) > 0:
 519                                         files_Del_List.update({entries[0]: elt})
 520
 521                 for elt in files_new:
 522                         if os.path.isfile(BASE_NEW + '/' + elt):
 523                                 FileName = path_leaf(elt)
 524                                 entries = re.split('[0-9]', FileName)
 525                                 #print('Entires under NEWfiles list after split  - %s %s - %s' % (FileName, entries[0], elt))
 526                                 if len(entries[0]) > 0:
 527                                         files_New_List.update({entries[0]: elt})
 528
 529                 for key, value in files_Del_List.iteritems():
 530                         #print('Key value pair -%s -%s' % (key, value))
 531                         if key in files_New_List:
 532                                 # this file is the same name in both!
 533                                 src_file = BASE_OLD + '/' + value
 534                                 dst_file = BASE_NEW + '/' + files_New_List[key]
 535                                 # we don't want to move hardlinks
 536                                 if ishardlink(src_file) or ishardlink(dst_file):
 537                                         logging.debug('Cannot diff as one of them is a hardlink')
 538                                 elif os.path.islink(src_file) or os.path.islink(dst_file):
 539                                         logging.debug('Cannot diff as one of them is Symlink')
 540                                 elif os.path.isdir(src_file) or os.path.isdir(dst_file):
 541                                         logging.debug('Cannot diff as one of them is dir')
 542                                 else:
 543                                         #Pick the best diff of same type and diff names
 544                                         files_renamed.append([files_New_List[key], value])
 545                                         files_removed.remove(value)
 546                                         files_new.remove(files_New_List[key])
 547
 548         '''
 549         Patch Section
 550                 Partition.txt contains Protocol for UPI
 551                 Types Supported: DIFFS, MOVES, NEWS, DELETES, SYMDIFFS, SYMNEWS.
 552         '''
 553         Sym_Diff_Cnt = 0
 554         Sym_New_Cnt = 0
 555         Hard_Diff_Cnt = 0
 556         Hard_New_Cnt = 0
 557         Del_Cnt = 0
 558         New_Cnt = 0
 559         Diff_Cnt = 0
 560         Move_Cnt = 0
 561         Verbatim_Cnt = 0
 562         SymLinkDoc = OUT_DIR + '/' + PART_NAME + SYMLINK_DOC_NAME
 563         HardLinkDoc = OUT_DIR + '/' + PART_NAME + HARDLINK_DOC_NAME
 564         Partition_Doc = open(OUT_DIR + '/' + PART_NAME + '.txt', 'w')
 565         Partition_Doc_SymLinks = open(SymLinkDoc, 'w')
 566         Partition_Doc_HardLinks = open(HardLinkDoc, "w")
 567
 568         print("writing diff'ed changed files...")
 569         for elt in files_changed:
 570                 dst_file = BASE_NEW + '/' + elt
 571                 src_file = BASE_OLD + '/' + elt
 572                 # Both files are symbolic links and they differ
 573                 if os.path.islink(dst_file) and os.path.islink(src_file):
 574                         # Both are symlinks and they differ
 575                         logging.debug(' File Changed is Link %s ' % dst_file)
 576                         patch = os.readlink(dst_file)
 577                         Sym_Diff_Cnt = Sym_Diff_Cnt + 1
 578                         Partition_Doc_SymLinks.write('SYM:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 579                         Update_Attr(elt, "SYM", File_Attributes, Sym_Attributes)
 580                 # Both are hardlinks and they differ (point to something different, new/changed file)
 581                 if elt in Old_hardlinks and elt in New_hardlinks:
 582                         if Old_hardlinks[elt] != New_hardlinks[elt] or New_hardlinks[elt] in files_changed or New_hardlinks[elt] in files_new:
 583                                 logging.debug('Hardlinks changed %s %s' % (src_file, dst_file))
 584                                 patch = New_hardlinks[elt]
 585                                 Hard_Diff_Cnt += 1
 586                                 Partition_Doc_HardLinks.write('HARD:DIFF:%s:%s:%s\n' % (elt, elt, patch))
 587                 # Both are NORMAL files and they differ
 588                 elif (not (os.path.islink(src_file) or os.path.islink(dst_file))) \
 589                         and (not (elt in Old_hardlinks or elt in New_hardlinks)) \
 590                         and os.path.isfile(dst_file) and os.path.isfile(src_file):
 591                         # Both are files and they differ
 592                         Diff_Cnt = Diff_Cnt + 1
 593                         patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt))
 594                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 595                         logging.debug(' File Differ %s %s' % (src_file, dst_file))
 596                         SS_UpdateSize(src_file, dst_file)
 597
 598                         FORMAT = "REG"
 599                         ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
 600                         if ret is not 0:
 601                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 602                                 files_new.append(elt)
 603                                 Diff_Cnt = Diff_Cnt - 1
 604                         else:
 605                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt, elt, hash_file(src_file), hash_file(dst_file), patchName))
 606
 607                         Update_Attr(elt, "FILE", File_Attributes, Sym_Attributes)
 608                 # Both differ but they are of diff types
 609                 else:
 610                         # Processing and updating partition txt file will be done under REMOVED case and NEW files case accordingly, we just make an entry here
 611                         files_removed.append(elt)
 612                         files_new.append(elt)
 613
 614         fdupes = find_dupes_list(BASE_OLD, BASE_NEW, files_removed, files_new, Old_hardlinks, New_hardlinks)
 615         for oldpath, newpath in fdupes.iteritems():
 616                 logging.info('Dupes %s -> %s' % (oldpath, newpath))
 617         for elt in files_removed:
 618                 src_file = BASE_OLD + '/' + elt
 619                 # If parent directory is deleted.. & del end not possible. (==> Moves should be done before deletes in ENGINE)
 620                 if src_file in fdupes.keys():
 621                         dst_file = BASE_NEW + '/' + fdupes[src_file]
 622                         logging.debug(' File Moved %s ==> %s' % (src_file, dst_file))
 623                         Move_Cnt = Move_Cnt + 1
 624                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt, fdupes[src_file], hash_file(src_file)))
 625                         files_removed.remove(elt)
 626                         files_new.remove(fdupes[src_file])
 627         # Should be placed after removing duplicates, else they will be filtered here.
 628         # loop shd b for all NEW files, rather than for all delete files (Current understanding)
 629         # First Step: Sort & Filter out unwanted files
 630         # Minimum condition used is,
 631         #       1. File name should match 70%
 632         #       2. Extensions should be same
 633         #       3. File name length shd b greater than 3 char
 634         #       4. As we are using sorting on file names, once file name does not match and R_Flag is set to true, we nee not check remaining files. So, will execute break.
 635         #       5. Should consider editdistance for RENAME LOGIC ==> TBD
 636         Base_DelList = files_removed[:]
 637         Base_NewList = files_new[:]
 638         DelList = sorted(Base_DelList, key=path_leaf)
 639         NewList = sorted(Base_NewList, key=path_leaf)
 640         logging.debug('Rename Logic before filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 641
 642         Filter1 = []
 643         Filter2 = []
 644         # Remove unwanted items which we cant make diff with for rename logic
 645         for file in DelList:
 646                 if os.path.islink(BASE_OLD + '/' + file):
 647                         continue
 648                 elif ishardlink(BASE_OLD + '/' + file):
 649                         continue
 650                 elif os.path.isdir(BASE_OLD + '/' + file):
 651                         continue
 652                 else:
 653                         Filter1.append(file)
 654                         #logging.debug('Sorted del list - %s' % (file))
 655
 656         DelList = Filter1
 657
 658         for file in NewList:
 659                 if os.path.islink(BASE_NEW + '/' + file):
 660                         continue
 661                 elif ishardlink(BASE_NEW + '/' + file):
 662                         continue
 663                 elif os.path.isdir(BASE_NEW + '/' + file):
 664                         continue
 665                 elif len(path_leaf(file)) <= 3:
 666                         logging.debug('Ignored for best picks -%s ' % (BASE_NEW + '/' + file))
 667                         continue
 668                 else:
 669                         Filter2.append(file)
 670         NewList = Filter2
 671         logging.debug('Rename Logic After filter: Delcount -%d NewCount -%d' % (len(DelList), len(NewList)))
 672
 673         for new_file in NewList:
 674                 R_Flag = 'FALSE'
 675                 DirPathNew = path_head(new_file)
 676                 FileNameNew = path_leaf(new_file)
 677                 DiffSize = 0
 678                 winning_patch_sz = os.path.getsize(BASE_NEW + '/' + new_file)
 679                 New_fs = winning_patch_sz
 680                 winning_file = ''
 681
 682                 for del_file in DelList:
 683                         FileNameOld = path_leaf(del_file)
 684                         if (FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and (os.path.splitext(FileNameNew)[1] == os.path.splitext(del_file)[1])):
 685                                 #winning_patch_sz = 0.9 * os.path.getsize(BASE_NEW+'/'+new_file)
 686                                 # Percentage difference between two file sizes is within 30%, then we consider for diff generation
 687                                 Del_fs = os.path.getsize(BASE_OLD + '/' + del_file)
 688                                 v1 = abs(New_fs - Del_fs)
 689                                 v2 = (New_fs + Del_fs) / 2
 690                                 if(v2 <= 0 or ((v1 / v2) * 100) > 30):
 691                                         logging.debug('Ignore diff generation New_fs - %d Del_Fs - %d' % (New_fs, Del_fs))
 692                                         continue
 693                                 logging.debug('I can compute diff between %s %s Del_Fs - %d New_Fs - %d' % (del_file, new_file, Del_fs, New_fs))
 694                                 R_Flag = 'TRUE'
 695                                 DiffSize = measure_two_filediffs(BASE_OLD + '/' + del_file, BASE_NEW + '/' + new_file)
 696                                 if (DiffSize < 0.8 * winning_patch_sz):
 697                                         winning_patch_sz = DiffSize
 698                                         winning_file = del_file
 699                         elif (not FileNameOld.startswith(FileNameNew[:len(FileNameNew) * 7 / 10]) and R_Flag == 'TRUE'):
 700                                 logging.debug('Because nex set of files will not have matching name - break @@ %s %s' % (del_file, new_file))
 701                                 break
 702                 if len(winning_file) > 0:
 703                         logging.debug('Best Pick -%s ==> %s [%d]' % (winning_file, new_file, DiffSize))
 704                         files_renamed.append([new_file, winning_file])
 705                         DelList.remove(winning_file)
 706                         files_removed.remove(winning_file)
 707                         files_new.remove(new_file)
 708
 709         #********************** Files should NOT be deleted for any such renames ***********************
 710
 711         if SUPPORT_RENAME == "TRUE":
 712                 for elt in files_renamed:
 713                         src_file = BASE_OLD + '/' + elt[1]
 714                         dst_file = BASE_NEW + '/' + elt[0]
 715                         Diff_Cnt = Diff_Cnt + 1
 716                         patchName = (DIFF_PREFIX + '%d_%s_' + PART_NAME + DIFF_SUFFIX) % (Diff_Cnt, path_leaf(elt[1]))
 717                         #patchName = (DIFF_PREFIX+'_%s'+DIFF_SUFFIX) % (path_leaf(elt[0]))
 718                         patchLoc = '%s/%s' % (OUT_DIR, patchName)
 719                         logging.debug(' File Renamed %s ==> %s' % (src_file, dst_file))
 720                         # Should be careful of renaming files??
 721                         # Should we consider measure_two_filediffs ?? so that patch size is NOT greater than actual file?
 722                         # What if folder path has numerics??
 723
 724                         if os.path.isdir(src_file) or os.path.isdir(dst_file):
 725                                 # This case never occurs??
 726                                 Partition_Doc.write('"%s" and "%s" renamed 0 0\n' % (elt[0], elt[1]))
 727                                 Update_Attr(elt[0], "FILE", File_Attributes, Sym_Attributes)
 728             # Make sure these files are PROPER and they shd NOT be symlinks
 729                         elif not (os.path.islink(src_file) or os.path.islink(dst_file)) \
 730                 and not (elt[0] in New_hardlinks or elt[1] in Old_hardlinks) \
 731                 and (os.path.isfile(src_file) and os.path.isfile(dst_file)):
 732                                 if filecmp.cmp(src_file, dst_file):
 733                                         Move_Cnt = Move_Cnt + 1
 734                                         Diff_Cnt = Diff_Cnt - 1
 735                                         Partition_Doc.write('MOVE:REG:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file)))
 736                                 else:
 737                                         FORMAT = "REG"
 738                                         ret = subprocess.call([DIFF_UTIL, src_file, dst_file, patchLoc])
 739                                         if ret is not 0:
 740                                                 logging.debug('Failed to create diff %d %s %s\n' % (ret, src_file, dst_file))
 741                                                 files_new.append(elt)
 742                                                 Diff_Cnt = Diff_Cnt - 1
 743                                         else:
 744                                                 Partition_Doc.write('DIFF:REG:%s:%s:%s:%s:%s\n' % (elt[1], elt[0], hash_file(src_file), hash_file(dst_file), patchName))
 745
 746                                 SS_UpdateSize(src_file, dst_file)
 747                                 Update_Attr(elt[0], "FILE", File_Attributes, Sym_Attributes)
 748
 749         # HANDLING VERBATIM - We Process NEWs and DELETEs for Verbatim list ONLY after processing duplicates & rename functionality.
 750         # So that, the rename functionality will NOT create PATCH instead of verbatims.
 751
 752         if SUPPORT_VERBATIM == "TRUE" and os.path.exists(VERBATIM_LIST):
 753                 with open(VERBATIM_LIST, 'r') as F_News:
 754                         lines = set(F_News.read().splitlines())
 755                 for line in lines:
 756                         if line not in files_new:
 757                                 if os.path.exists(BASE_NEW + '/' + line):
 758                                         files_new.append(line)
 759                                         Verbatim_Cnt = Verbatim_Cnt + 1
 760                                         logging.debug("Added to list of verbatims -%s" % BASE_NEW + '/' + line)
 761
 762         for elt in files_removed:
 763                 # if files are part of patches after renaming, we shd remove them as part of removed.
 764                 src_file = BASE_OLD + '/' + elt
 765                 if os.path.islink(src_file):
 766                         Partition_Doc.write('DEL:SYM:%s\n' % (elt))
 767                 elif elt in Old_hardlinks:
 768                         Partition_Doc.write('DEL:HARD:%s\n' % (elt))
 769                 elif os.path.isdir(src_file):
 770                         # If we change to DIR TYPE, then the same token should be modified on UA also and SHA should be accordingly passed.
 771                         Partition_Doc.write('DEL:REG:%s:NA\n' % (elt))
 772                 else:
 773                         Partition_Doc.write('DEL:REG:%s:%s\n' % (elt, hash_file(src_file)))
 774                 logging.debug(' File Deleted %s' % src_file)
 775                 Del_Cnt = Del_Cnt + 1
 776
 777         Dir_removed.sort(reverse=True)
 778         for elt in Dir_removed:
 779                 # if Dir is empty, add it to the removed list.
 780                 src_file = BASE_OLD + '/' + elt
 781                 # Irrespective of weather files are MOVED or DIFF'ed, we can delete the folders. This action can be performed at the end.
 782                 # It covers symlinks also, as NEW symlinks cannot point to NON existant folders of TARGET (NEW binary)
 783                 if os.path.isdir(src_file):
 784                         Partition_Doc.write('DEL:END:%s\n' % (elt))
 785                         Del_Cnt = Del_Cnt + 1
 786                         logging.debug(' Dir Deleted- %s' % src_file)
 787
 788         for elt in files_new:
 789                 dst_file = BASE_NEW + '/' + elt
 790                 newfiles_dest_path = 'run/upgrade-sysroot/'
 791                 try:
 792                         ensure_dir_exists(newfiles_dest_path)
 793                 except FileExistsError as exc:
 794                         logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path)
 795                         raise exc
 796                 if os.path.islink(dst_file):
 797                         patch = os.readlink(dst_file)
 798                         logging.debug(' File New Links %s' % elt)
 799                         Partition_Doc_SymLinks.write('SYM:NEW:%s:%s\n' % (elt, patch))
 800                         # What if this is only a new sym link and folder already exists??? Should recheck
 801                         destpath = newfiles_dest_path + elt
 802                         if not os.path.exists(path_head(destpath)):
 803                                 os.makedirs(path_head(destpath))
 804                                 logging.info('New SymLink - Adding missing Dir')
 805                         Update_Attr(elt, "SYM", File_Attributes, Sym_Attributes)
 806                         Sym_New_Cnt = Sym_New_Cnt + 1
 807                 elif elt in New_hardlinks:
 808                         patch = New_hardlinks[elt]
 809                         logging.debug('File new hardlink %s' % elt)
 810                         Partition_Doc_HardLinks.write('HARD:NEW:%s:%s\n' %(elt, patch))
 811                         destpath = newfiles_dest_path + elt
 812                         if not os.path.exists(path_head(destpath)):
 813                                 os.makedirs(path_head(destpath))
 814                                 logging.info('New hardlink - Adding missing Dir')
 815                         Hard_New_Cnt += 1
 816                 elif os.path.isdir(dst_file):  # We create just empty directory here
 817                         destpath = newfiles_dest_path + elt
 818                         if not os.path.exists(destpath):
 819                                 os.makedirs(destpath)
 820                                 logging.debug(' File New Dir %s' % destpath)
 821                                 New_Cnt = New_Cnt + 1
 822                 else:
 823                         New_Cnt = New_Cnt + 1
 824                         destpath = newfiles_dest_path + elt
 825                         destdir = os.path.dirname(destpath)
 826                         logging.debug('New files - %s ==> %s' % (dst_file, destdir))
 827
 828                         if not os.path.isdir(destdir):
 829                                 try:
 830                                         os.makedirs(destdir)
 831                                 except Exception as exc:
 832                                         logging.critical('Error in NEW files DIR entry -%s' % destdir)
 833                                         raise exc
 834
 835                         try:
 836                                 if not stat.S_ISFIFO(os.stat(dst_file).st_mode):
 837                                         shutil.copy2(dst_file, destpath)
 838                                         logging.debug('New files copied from- %s to- %s' % (dst_file, destpath))
 839                         except Exception as exc:
 840                                 logging.critical('Error in NEW files entry -%s -%s' % (dst_file, destpath))
 841                                 raise exc
 842                         Update_Attr(elt, "FILE", File_Attributes, Sym_Attributes)
 843
 844         for elt in Dir_Added:
 845                 newfiles_dest_path = 'run/upgrade-sysroot/'
 846                 try:
 847                         ensure_dir_exists(newfiles_dest_path)
 848                 except FileExistsError as exc:
 849                         logging.error('Directory %s used by this script is already an existing file' % newfiles_dest_path)
 850                         raise exc
 851                 destpath = newfiles_dest_path + elt
 852                 if not os.path.exists(destpath):
 853                         os.makedirs(destpath)
 854                         logging.debug(' DirList New Dir %s' % destpath)
 855                         New_Cnt = New_Cnt + 1
 856
 857         # Base directory should be system
 858         print 'Compressing New files'
 859         if (New_Cnt > 0 or Sym_New_Cnt > 0):
 860                 WorkingDir = os.getcwd()
 861                 os.chdir(os.getcwd() + "/" + NEW_FILES_PATH)
 862                 logging.info('Curr Working Dir - %s' % os.getcwd())
 863                 os.system(ZIPUTIL + NEW_FILES_PATH + " >> " + LOGFILE)
 864                 shutil.move(NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR)
 865                 # New file size?? cos, we extract system.7z from delta.tar and then proceed with decompression
 866                 SS_UpdateSize(WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME, WorkingDir + "/" + OUT_DIR + "/" + NEW_FILES_ZIP_NAME)
 867                 os.chdir(WorkingDir)
 868                 shutil.rmtree(NEW_FILES_PATH)
 869                 # use 7z a system.7z ./*
 870
 871         #logging.info('%d Dir to be removed' % len(Dir_removed))
 872         logging.info('%d files unchanged' % len(files_unchanged))
 873         logging.info('%d files files_renamed' % len(files_renamed))
 874         logging.info('%d files NEW' % len(files_new))
 875         logging.info('%d File attr' % len(File_Attributes))
 876         logging.info('%d Sym attr' % len(Sym_Attributes))
 877         logging.info('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d HardDiffs-%d HardNews-%d Verbatim -%d\n' % \
 878                 (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Hard_Diff_Cnt, Hard_New_Cnt, Verbatim_Cnt))
 879         print('PaTcHCoUnT:Diffs-%d Moves-%d News-%d Delets-%d SymDiffs-%d SymNews-%d HardDiffs-%d HardNews-%d Verbatim -%d\n' % \
 880                 (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Hard_Diff_Cnt, Hard_New_Cnt, Verbatim_Cnt))
 881
 882         # There could be duplicates, TODO, can check before adding..
 883         ATTR_FILE_D = open(ATTR_FILE, 'a+')
 884         for elt in File_Attributes:
 885                 ATTR_FILE_D.write(elt)
 886         for elt in Sym_Attributes:
 887                 ATTR_FILE_D.write(elt)
 888
 889         ATTR_FILE_D.close()
 890
 891         Partition_Doc_SymLinks.close()
 892         Partition_Doc_HardLinks.close()
 893         Partition_Read_SymLinks = open(SymLinkDoc, 'r+')
 894         Partition_Read_HardLinks = open(HardLinkDoc, 'r+')
 895         Partition_Doc.write(Partition_Read_SymLinks.read())
 896         for line in reversed(Partition_Read_HardLinks.readlines()):
 897                 Partition_Doc.write(line)
 898         Partition_Doc.write('PaTcHCoUnT:%d %d %d %d %d %d %d %d\n' % \
 899                 (Diff_Cnt, Move_Cnt, New_Cnt, Del_Cnt, Sym_Diff_Cnt, Sym_New_Cnt, Hard_Diff_Cnt, Hard_New_Cnt))
 900         Partition_Read_SymLinks.close()
 901         Partition_Read_HardLinks.close()
 902         Partition_Doc.close()
 903         os.remove(SymLinkDoc)
 904         os.remove(HardLinkDoc)
 905
 906         if Diff_Cnt + Move_Cnt + New_Cnt + Del_Cnt + Sym_Diff_Cnt + Sym_New_Cnt + Verbatim_Cnt + Hard_Diff_Cnt + \
 907                 Hard_New_Cnt + os.path.getsize(ATTR_FILE) == 0:
 908                 print('No Delta Generated for %s - %s' % (PART_NAME, OUT_DIR))
 909                 logging.info('No Delta Generated for %s' % PART_NAME)
 910                 shutil.rmtree(OUT_DIR)
 911
 912
 913 def IsSymlink(info):
 914         return (info.external_attr >> 16) == 0120777
 915
 916
 917 def NewFiles(src, dest):
 918         print src, dest
 919         subprocess.call(['cp', '-rp', src, dest])
 920         #try:
 921                 #shutil.copytree(src, dest)
 922         #except OSError as e:
 923                 # If the error was caused because the source wasn't a directory
 924                 #if e.errno == errno.ENOTDIR:
 925                         #shutil.copy2(src, dest)
 926                 #else:
 927                         #print('Directory not copied. Error: %s' % e)
 928
 929
 930 def measure_two_filediffs(src, dst):
 931         patchLoc = 'temp.patch'
 932         # TODO ensure this is excepts an error
 933         subprocess.call([DIFF_UTIL, src, dst, patchLoc])
 934         result_size = os.path.getsize(patchLoc)
 935         os.remove(patchLoc)
 936         return result_size
 937
 938
 939 def ishardlink(path):
 940         if os.stat(path).st_nlink > 1:
 941                 return True
 942         return False
 943
 944
 945 def get_inode(path):
 946         return os.stat(path).st_ino
 947
 948
 949 def get_hardlinks(base):
 950         hardlinks_dict = {}
 951         inodes_dict = {}
 952
 953         for root, direcotories, files in os.walk(base, topdown=True, followlinks=False):
 954                 for file in sorted(files):
 955                         file_name = os.path.join(root, file)
 956                         if not os.path.islink(file_name) and ishardlink(file_name):
 957                                 inode = get_inode(file_name)
 958                                 rel_path = os.path.relpath(file_name, base)
 959                                 if inode not in inodes_dict:
 960                                         inodes_dict[inode] = rel_path
 961                                 else:
 962                                         hardlinks_dict[rel_path] = inodes_dict[inode]
 963
 964         return hardlinks_dict
 965
 966
 967 def Get_Files(path):
 968         all_files = []
 969         all_dirs = []
 970
 971         for root, directories, filenames in os.walk(path, topdown=False, followlinks=False):
 972                 for directory in directories:
 973                         #DirName = os.path.join(root+'/',directory)
 974                         DirName = os.path.join(root, directory)
 975                         if os.path.islink(DirName):
 976                                 logging.debug('This is symlink pointing to dir -%s' % DirName)
 977                                 all_files.append(os.path.relpath(DirName, path))
 978                         elif not os.listdir(DirName):
 979                                 #print('*****Empty Directory******* -%s', DirName)
 980                                 # This should NOT be appended ??? Empty dir shd b considered
 981                                 all_dirs.append(os.path.relpath(DirName, path))
 982                         else:
 983                                 all_dirs.append(os.path.relpath(DirName, path))
 984                 for filename in filenames:
 985                         FileName = os.path.join(root, filename)
 986                         all_files.append(os.path.relpath(FileName, path))
 987
 988         all_files.sort()
 989         all_dirs.sort()
 990         return all_files, all_dirs
 991
 992
 993 USAGE_DOCSTRING = """
 994         Generate Delta using BASEOLD AND BASE NEW
 995         Attributes is optional
 996         Usage: CreatePatch.py UPDATE_TYPE PARTNAME OLDBASE NEWBASE OUTFOLDER
 997 """
 998
 999
1000 def Usage(docstring):
1001         print docstring.rstrip("\n")
1002         print COMMON_DOCSTRING
1003
1004
1005 if __name__ == '__main__':
1006         main()