src/third_party/skia/tools/svndiff.py

   1 #!/usr/bin/python
   2 '''
   3 Copyright 2012 Google Inc.
   4
   5 Use of this source code is governed by a BSD-style license that can be
   6 found in the LICENSE file.
   7 '''
   8
   9 '''
  10 Generates a visual diff of all pending changes in the local SVN (or git!)
  11 checkout.
  12
  13 Launch with --help to see more information.
  14
  15 TODO(epoger): Now that this tool supports either git or svn, rename it.
  16 TODO(epoger): Fix indentation in this file (2-space indents, not 4-space).
  17 '''
  18
  19 # common Python modules
  20 import optparse
  21 import os
  22 import posixpath
  23 import re
  24 import shutil
  25 import subprocess
  26 import sys
  27 import tempfile
  28 import urllib2
  29
  30 # Imports from within Skia
  31 #
  32 # We need to add the 'gm' directory, so that we can import gm_json.py within
  33 # that directory.  That script allows us to parse the actual-results.json file
  34 # written out by the GM tool.
  35 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
  36 # so any dirs that are already in the PYTHONPATH will be preferred.
  37 #
  38 # This assumes that the 'gm' directory has been checked out as a sibling of
  39 # the 'tools' directory containing this script, which will be the case if
  40 # 'trunk' was checked out as a single unit.
  41 GM_DIRECTORY = os.path.realpath(
  42     os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
  43 if GM_DIRECTORY not in sys.path:
  44     sys.path.append(GM_DIRECTORY)
  45 import gm_json
  46 import jsondiff
  47 import svn
  48
  49 USAGE_STRING = 'Usage: %s [options]'
  50 HELP_STRING = '''
  51
  52 Generates a visual diff of all pending changes in the local SVN/git checkout.
  53
  54 This includes a list of all files that have been added, deleted, or modified
  55 (as far as SVN/git knows about).  For any image modifications, pixel diffs will
  56 be generated.
  57
  58 '''
  59
  60 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
  61
  62 TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir)
  63
  64 OPTION_DEST_DIR = '--dest-dir'
  65 OPTION_PATH_TO_SKDIFF = '--path-to-skdiff'
  66 OPTION_SOURCE_DIR = '--source-dir'
  67
  68 def RunCommand(command):
  69     """Run a command, raising an exception if it fails.
  70
  71     @param command the command as a single string
  72     """
  73     print 'running command [%s]...' % command
  74     retval = os.system(command)
  75     if retval is not 0:
  76         raise Exception('command [%s] failed' % command)
  77
  78 def FindPathToSkDiff(user_set_path=None):
  79     """Return path to an existing skdiff binary, or raise an exception if we
  80     cannot find one.
  81
  82     @param user_set_path if None, the user did not specify a path, so look in
  83            some likely places; otherwise, only check at this path
  84     """
  85     if user_set_path is not None:
  86         if os.path.isfile(user_set_path):
  87             return user_set_path
  88         raise Exception('unable to find skdiff at user-set path %s' %
  89                         user_set_path)
  90     trunk_path = os.path.join(os.path.dirname(__file__), os.pardir)
  91
  92     extension = ''
  93     if os.name is 'nt':
  94         extension = '.exe'
  95
  96     possible_paths = [os.path.join(trunk_path, 'out', 'Release',
  97                                     'skdiff' + extension),
  98                       os.path.join(trunk_path, 'out', 'Debug',
  99                                    'skdiff' + extension)]
 100     for try_path in possible_paths:
 101         if os.path.isfile(try_path):
 102             return try_path
 103     raise Exception('cannot find skdiff in paths %s; maybe you need to '
 104                     'specify the %s option or build skdiff?' % (
 105                         possible_paths, OPTION_PATH_TO_SKDIFF))
 106
 107 def _DownloadUrlToFile(source_url, dest_path):
 108     """Download source_url, and save its contents to dest_path.
 109     Raises an exception if there were any problems."""
 110     try:
 111         reader = urllib2.urlopen(source_url)
 112         writer = open(dest_path, 'wb')
 113         writer.write(reader.read())
 114         writer.close()
 115     except BaseException as e:
 116         raise Exception(
 117             '%s: unable to download source_url %s to dest_path %s' % (
 118                 e, source_url, dest_path))
 119
 120 def _CreateGSUrl(imagename, hash_type, hash_digest):
 121     """Return the HTTP URL we can use to download this particular version of
 122     the actually-generated GM image with this imagename.
 123
 124     imagename: name of the test image, e.g. 'perlinnoise_msaa4.png'
 125     hash_type: string indicating the hash type used to generate hash_digest,
 126                e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
 127     hash_digest: the hash digest of the image to retrieve
 128     """
 129     return gm_json.CreateGmActualUrl(
 130         test_name=IMAGE_FILENAME_RE.match(imagename).group(1),
 131         hash_type=hash_type,
 132         hash_digest=hash_digest)
 133
 134 def _CallJsonDiff(old_json_path, new_json_path,
 135                   old_flattened_dir, new_flattened_dir,
 136                   filename_prefix):
 137     """Using jsondiff.py, write the images that differ between two GM
 138     expectations summary files (old and new) into old_flattened_dir and
 139     new_flattened_dir.
 140
 141     filename_prefix: prefix to prepend to filenames of all images we write
 142         into the flattened directories
 143     """
 144     json_differ = jsondiff.GMDiffer()
 145     diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path,
 146                                              newfile=new_json_path)
 147     print 'Downloading %d before-and-after image pairs...' % len(diff_dict)
 148     for (imagename, results) in diff_dict.iteritems():
 149         # TODO(epoger): Currently, this assumes that all images have been
 150         # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5
 151
 152         old_checksum = results['old']
 153         if old_checksum:
 154             old_image_url = _CreateGSUrl(
 155                 imagename=imagename,
 156                 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
 157                 hash_digest=old_checksum)
 158             _DownloadUrlToFile(
 159                 source_url=old_image_url,
 160                 dest_path=os.path.join(old_flattened_dir,
 161                                        filename_prefix + imagename))
 162
 163         new_checksum = results['new']
 164         if new_checksum:
 165             new_image_url = _CreateGSUrl(
 166                 imagename=imagename,
 167                 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5,
 168                 hash_digest=new_checksum)
 169             _DownloadUrlToFile(
 170                 source_url=new_image_url,
 171                 dest_path=os.path.join(new_flattened_dir,
 172                                        filename_prefix + imagename))
 173
 174 def _RunCommand(args):
 175     """Run a command (from self._directory) and return stdout as a single
 176     string.
 177
 178     @param args a list of arguments
 179     """
 180     proc = subprocess.Popen(args,
 181                             stdout=subprocess.PIPE,
 182                             stderr=subprocess.PIPE)
 183     (stdout, stderr) = proc.communicate()
 184     if proc.returncode is not 0:
 185         raise Exception('command "%s" failed: %s' % (args, stderr))
 186     return stdout
 187
 188 def _GitGetModifiedFiles():
 189     """Returns a list of locally modified files within the current working dir.
 190
 191     TODO(epoger): Move this into a git utility package?
 192     """
 193     return _RunCommand(['git', 'ls-files', '-m']).splitlines()
 194
 195 def _GitExportBaseVersionOfFile(file_within_repo, dest_path):
 196     """Retrieves a copy of the base version of a file within the repository.
 197
 198     @param file_within_repo path to the file within the repo whose base
 199            version you wish to obtain
 200     @param dest_path destination to which to write the base content
 201
 202     TODO(epoger): Move this into a git utility package?
 203     """
 204     # TODO(epoger): Replace use of "git show" command with lower-level git
 205     # commands?  senorblanco points out that "git show" is a "porcelain"
 206     # command, intended for human use, as opposed to the "plumbing" commands
 207     # generally more suitable for scripting.  (See
 208     # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain )
 209     #
 210     # For now, though, "git show" is the most straightforward implementation
 211     # I could come up with.  I tried using "git cat-file", but I had trouble
 212     # getting it to work as desired.
 213     # Note that git expects / rather than \ as a path separator even on
 214     # windows.
 215     args = ['git', 'show', posixpath.join('HEAD:.', file_within_repo)]
 216     with open(dest_path, 'wb') as outfile:
 217         proc = subprocess.Popen(args, stdout=outfile)
 218         proc.communicate()
 219         if proc.returncode is not 0:
 220             raise Exception('command "%s" failed' % args)
 221
 222 def SvnDiff(path_to_skdiff, dest_dir, source_dir):
 223     """Generates a visual diff of all pending changes in source_dir.
 224
 225     @param path_to_skdiff
 226     @param dest_dir existing directory within which to write results
 227     @param source_dir
 228     """
 229     # Validate parameters, filling in default values if necessary and possible.
 230     path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff))
 231     if not dest_dir:
 232         dest_dir = tempfile.mkdtemp()
 233     dest_dir = os.path.abspath(dest_dir)
 234
 235     os.chdir(source_dir)
 236     svn_repo = svn.Svn('.')
 237     using_svn = True
 238     try:
 239       svn_repo.GetInfo()
 240     except:
 241       using_svn = False
 242
 243     # Prepare temporary directories.
 244     modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened')
 245     original_flattened_dir = os.path.join(dest_dir, 'original_flattened')
 246     diff_dir = os.path.join(dest_dir, 'diffs')
 247     for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] :
 248         shutil.rmtree(dir, ignore_errors=True)
 249         os.mkdir(dir)
 250
 251     # Get a list of all locally modified (including added/deleted) files,
 252     # descending subdirectories.
 253     if using_svn:
 254         modified_file_paths = svn_repo.GetFilesWithStatus(
 255             svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
 256     else:
 257         modified_file_paths = _GitGetModifiedFiles()
 258
 259     # For each modified file:
 260     # 1. copy its current contents into modified_flattened_dir
 261     # 2. copy its original contents into original_flattened_dir
 262     for modified_file_path in modified_file_paths:
 263         if modified_file_path.endswith('.json'):
 264             # Special handling for JSON files, in the hopes that they
 265             # contain GM result summaries.
 266             original_file = tempfile.NamedTemporaryFile(delete = False)
 267             original_file.close()
 268             if using_svn:
 269                 svn_repo.ExportBaseVersionOfFile(
 270                     modified_file_path, original_file.name)
 271             else:
 272                 _GitExportBaseVersionOfFile(
 273                     modified_file_path, original_file.name)
 274             modified_dir = os.path.dirname(modified_file_path)
 275             platform_prefix = (re.sub(re.escape(os.sep), '__',
 276                                       os.path.splitdrive(modified_dir)[1])
 277                               + '__')
 278             _CallJsonDiff(old_json_path=original_file.name,
 279                           new_json_path=modified_file_path,
 280                           old_flattened_dir=original_flattened_dir,
 281                           new_flattened_dir=modified_flattened_dir,
 282                           filename_prefix=platform_prefix)
 283             os.remove(original_file.name)
 284         else:
 285             dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path)
 286             # If the file had STATUS_DELETED, it won't exist anymore...
 287             if os.path.isfile(modified_file_path):
 288                 shutil.copyfile(modified_file_path,
 289                                 os.path.join(modified_flattened_dir,
 290                                              dest_filename))
 291             if using_svn:
 292                 svn_repo.ExportBaseVersionOfFile(
 293                     modified_file_path,
 294                     os.path.join(original_flattened_dir, dest_filename))
 295             else:
 296                 _GitExportBaseVersionOfFile(
 297                     modified_file_path,
 298                     os.path.join(original_flattened_dir, dest_filename))
 299
 300     # Run skdiff: compare original_flattened_dir against modified_flattened_dir
 301     RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir,
 302                                 modified_flattened_dir, diff_dir))
 303     print '\nskdiff results are ready in file://%s/index.html' % diff_dir
 304
 305 def RaiseUsageException():
 306     raise Exception('%s\nRun with --help for more detail.' % (
 307         USAGE_STRING % __file__))
 308
 309 def Main(options, args):
 310     """Allow other scripts to call this script with fake command-line args.
 311     """
 312     num_args = len(args)
 313     if num_args != 0:
 314         RaiseUsageException()
 315     SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir,
 316             source_dir=options.source_dir)
 317
 318 if __name__ == '__main__':
 319     parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
 320     parser.add_option(OPTION_DEST_DIR,
 321                       action='store', type='string', default=None,
 322                       help='existing directory within which to write results; '
 323                       'if not set, will create a temporary directory which '
 324                       'will remain in place after this script completes')
 325     parser.add_option(OPTION_PATH_TO_SKDIFF,
 326                       action='store', type='string', default=None,
 327                       help='path to already-built skdiff tool; if not set, '
 328                       'will search for it in typical directories near this '
 329                       'script')
 330     parser.add_option(OPTION_SOURCE_DIR,
 331                       action='store', type='string',
 332                       default=os.path.join('expectations', 'gm'),
 333                       help='root directory within which to compare all ' +
 334                       'files; defaults to "%default"')
 335     (options, args) = parser.parse_args()
 336     Main(options, args)