Tools/Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Rebaselining tool that automatically produces baselines for all platforms.
  31
  32 The script does the following for each platform specified:
  33   1. Compile a list of tests that need rebaselining.
  34   2. Download test result archive from buildbot for the platform.
  35   3. Extract baselines from the archive file for all identified files.
  36   4. Add new baselines to SVN repository.
  37   5. For each test that has been rebaselined, remove this platform option from
  38      the test in test_expectation.txt. If no other platforms remain after
  39      removal, delete the rebaselined test from the file.
  40
  41 At the end, the script generates a html that compares old and new baselines.
  42 """
  43
  44 from __future__ import with_statement
  45
  46 import copy
  47 import logging
  48 import optparse
  49 import re
  50 import sys
  51 import time
  52
  53 from webkitpy.common.checkout import scm
  54 from webkitpy.common.system import zipfileset
  55 from webkitpy.common.system import path
  56 from webkitpy.common.system import urlfetcher
  57 from webkitpy.common.system.executive import ScriptError
  58 from webkitpy.common.host import Host
  59
  60 from webkitpy.layout_tests.port.factory import PortFactory
  61 from webkitpy.layout_tests import read_checksum_from_png
  62 from webkitpy.layout_tests.models import test_expectations
  63
  64
  65 _log = logging.getLogger(__name__)
  66
  67 BASELINE_SUFFIXES = ('.txt', '.png', '.checksum')
  68
  69 ARCHIVE_DIR_NAME_DICT = {
  70     'chromium-win-win7': 'Webkit_Win7',
  71     'chromium-win-vista': 'Webkit_Vista',
  72     'chromium-win-xp': 'Webkit_Win',
  73     'chromium-mac-leopard': 'Webkit_Mac10_5',
  74     'chromium-mac-snowleopard': 'Webkit_Mac10_6',
  75     'chromium-cg-mac-leopard': 'Webkit_Mac10_5__CG_',
  76     'chromium-cg-mac-snowleopard': 'Webkit_Mac10_6__CG_',
  77     'chromium-linux-x86': 'Webkit_Linux_32',
  78     'chromium-linux-x86_64': 'Webkit_Linux',
  79     'chromium-gpu-mac-snowleopard': 'Webkit_Mac10_6_-_GPU',
  80     'chromium-gpu-win-xp': 'Webkit_Win_-_GPU',
  81     'chromium-gpu-win-win7': 'Webkit_Win7_-_GPU',
  82     'chromium-gpu-linux-x86_64': 'Webkit_Linux_-_GPU',
  83     'chromium-gpu-linux-x86': 'Webkit_Linux_32_-_GPU',
  84 }
  85
  86
  87 def log_dashed_string(text, platform=None, logging_level=logging.DEBUG):
  88     """Log text message with dashes on both sides."""
  89     msg = text
  90     if platform:
  91         msg += ': ' + platform
  92     if len(msg) < 78:
  93         dashes = '-' * ((78 - len(msg)) / 2)
  94         msg = '%s %s %s' % (dashes, msg, dashes)
  95     _log.log(logging_level, msg)
  96
  97
  98 def setup_html_directory(filesystem, parent_directory):
  99     """Setup the directory to store html results.
 100
 101        All html related files are stored in the "rebaseline_html" subdirectory of
 102        the parent directory. The path to the created directory is returned.
 103     """
 104
 105     if not parent_directory:
 106         parent_directory = str(filesystem.mkdtemp())
 107     else:
 108         filesystem.maybe_make_directory(parent_directory)
 109
 110     html_directory = filesystem.join(parent_directory, 'rebaseline_html')
 111     _log.debug('Html directory: "%s"', html_directory)
 112
 113     if filesystem.exists(html_directory):
 114         filesystem.rmtree(html_directory)
 115         _log.debug('Deleted html directory: "%s"', html_directory)
 116
 117     filesystem.maybe_make_directory(html_directory)
 118     return html_directory
 119
 120
 121 def get_result_file_fullpath(filesystem, html_directory, baseline_filename, platform,
 122                              result_type):
 123     """Get full path of the baseline result file.
 124
 125     Args:
 126       filesystem: wrapper object
 127       html_directory: directory that stores the html related files.
 128       baseline_filename: name of the baseline file.
 129       platform: win, linux or mac
 130       result_type: type of the baseline result: '.txt', '.png'.
 131
 132     Returns:
 133       Full path of the baseline file for rebaselining result comparison.
 134     """
 135
 136     base, ext = filesystem.splitext(baseline_filename)
 137     result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
 138     fullpath = filesystem.join(html_directory, result_filename)
 139     _log.debug('  Result file full path: "%s".', fullpath)
 140     return fullpath
 141
 142
 143 class Rebaseliner(object):
 144     """Class to produce new baselines for a given platform."""
 145
 146     REVISION_REGEX = r'<a href=\"(\d+)/\">'
 147
 148     def __init__(self, running_port, target_port, platform, options, url_fetcher, zip_factory, scm, logged_before=False):
 149         """
 150         Args:
 151             running_port: the Port the script is running on.
 152             target_port: the Port the script uses to find port-specific
 153                 configuration information like the test_expectations.txt
 154                 file location and the list of test platforms.
 155             platform: the test platform to rebaseline
 156             options: the command-line options object.
 157             url_fetcher: object that can fetch objects from URLs
 158             zip_factory: optional object that can fetch zip files from URLs
 159             scm: scm object for adding new baselines
 160             logged_before: whether the previous running port logged anything.
 161         """
 162         self._platform = platform
 163         self._options = options
 164         self._port = running_port
 165         self._filesystem = running_port._filesystem
 166         self._target_port = target_port
 167
 168         # FIXME: This should get its PortFactory from a Host object.
 169         # Note: using running_port.executive, running_port.user since we can't get them from a host.
 170         self._rebaseline_port = PortFactory().get(platform, options, filesystem=self._filesystem, executive=running_port.executive, user=running_port.user)
 171         self._rebaselining_tests = set()
 172         self._rebaselined_tests = []
 173         self._logged_before = logged_before
 174         self.did_log = False
 175
 176         # Create tests and expectations helper which is used to:
 177         #   -. compile list of tests that need rebaselining.
 178         #   -. update the tests in test_expectations file after rebaseline
 179         #      is done.
 180         expectations_str = self._rebaseline_port.test_expectations()
 181         self._test_expectations = test_expectations.TestExpectations(
 182             self._rebaseline_port, None, expectations_str, self._rebaseline_port.test_configuration(), False)
 183         self._url_fetcher = url_fetcher
 184         self._zip_factory = zip_factory
 185         self._scm = scm
 186
 187     def run(self):
 188         """Run rebaseline process."""
 189
 190         log_dashed_string('Compiling rebaselining tests', self._platform, logging.DEBUG)
 191         if not self._compile_rebaselining_tests():
 192             return False
 193         if not self._rebaselining_tests:
 194             return True
 195
 196         self.did_log = True
 197         log_dashed_string('Downloading archive', self._platform, logging.DEBUG)
 198         archive_file = self._download_buildbot_archive()
 199         _log.debug('')
 200         if not archive_file:
 201             _log.error('No archive found.')
 202             return False
 203
 204         log_dashed_string('Extracting and adding new baselines', self._platform, logging.DEBUG)
 205         self._extract_and_add_new_baselines(archive_file)
 206         archive_file.close()
 207
 208         log_dashed_string('Updating rebaselined tests in file', self._platform)
 209
 210         if len(self._rebaselining_tests) != len(self._rebaselined_tests):
 211             _log.debug('')
 212             _log.debug('NOT ALL TESTS WERE REBASELINED.')
 213             _log.debug('  Number marked for rebaselining: %d', len(self._rebaselining_tests))
 214             _log.debug('  Number actually rebaselined: %d', len(self._rebaselined_tests))
 215             _log.info('')
 216             return False
 217
 218         _log.debug('  All tests needing rebaselining were successfully rebaselined.')
 219         _log.info('')
 220         return True
 221
 222     def remove_rebaselining_expectations(self, tests, backup):
 223         """if backup is True, we backup the original test expectations file."""
 224         new_expectations = self._test_expectations.remove_rebaselined_tests(tests)
 225         path = self._target_port.path_to_test_expectations_file()
 226         if backup:
 227             date_suffix = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
 228             backup_file = '%s.orig.%s' % (path, date_suffix)
 229             if self._filesystem.exists(backup_file):
 230                 self._filesystem.remove(backup_file)
 231             _log.debug('Saving original file to "%s"', backup_file)
 232             self._filesystem.move(path, backup_file)
 233
 234         self._filesystem.write_text_file(path, new_expectations)
 235         # self._scm.add(path)
 236
 237     def get_rebaselined_tests(self):
 238         return self._rebaselined_tests
 239
 240     def _compile_rebaselining_tests(self):
 241         """Compile list of tests that need rebaselining for the platform.
 242
 243         Returns:
 244           False if reftests are wrongly marked as 'needs rebaselining' or True
 245         """
 246
 247         self._rebaselining_tests = self._test_expectations.get_rebaselining_failures()
 248         if not self._rebaselining_tests:
 249             _log.info('%s: No tests to rebaseline.', self._platform)
 250             return True
 251
 252         fs = self._target_port._filesystem
 253         for test in self._rebaselining_tests:
 254             test_abspath = self._target_port.abspath_for_test(test)
 255             if (fs.exists(self._target_port.reftest_expected_filename(test_abspath)) or
 256                 fs.exists(self._target_port.reftest_expected_mismatch_filename(test_abspath))):
 257                 _log.error('%s seems to be a reftest. We can not rebase for reftests.', test)
 258                 self._rebaselining_tests = set()
 259                 return False
 260
 261         if not self._logged_before:
 262             _log.info('')
 263         _log.info('%s: Rebaselining %d tests:', self._platform, len(self._rebaselining_tests))
 264         test_no = 1
 265         for test in self._rebaselining_tests:
 266             _log.debug('  %d: %s', test_no, test)
 267             test_no += 1
 268
 269         return True
 270
 271     def _get_latest_revision(self, url):
 272         """Get the latest layout test revision number from buildbot.
 273
 274         Args:
 275           url: Url to retrieve layout test revision numbers.
 276
 277         Returns:
 278           latest revision or
 279           None on failure.
 280         """
 281
 282         _log.debug('Url to retrieve revision: "%s"', url)
 283
 284         content = self._url_fetcher.fetch(url)
 285
 286         revisions = re.findall(self.REVISION_REGEX, content)
 287         if not revisions:
 288             _log.error('Failed to find revision, content: "%s"', content)
 289             return None
 290
 291         revisions.sort(key=int)
 292         _log.debug('  Latest revision: %s', revisions[len(revisions) - 1])
 293         return revisions[len(revisions) - 1]
 294
 295     def _get_archive_dir_name(self, platform):
 296         """Get name of the layout test archive directory.
 297
 298         Returns:
 299           Directory name or
 300           None on failure
 301         """
 302
 303         if platform in ARCHIVE_DIR_NAME_DICT:
 304             return ARCHIVE_DIR_NAME_DICT[platform]
 305         else:
 306             _log.error('Cannot find platform key %s in archive '
 307                        'directory name dictionary', platform)
 308             return None
 309
 310     def _get_archive_url(self):
 311         """Generate the url to download latest layout test archive.
 312
 313         Returns:
 314           Url to download archive or
 315           None on failure
 316         """
 317
 318         if self._options.force_archive_url:
 319             return self._options.force_archive_url
 320
 321         dir_name = self._get_archive_dir_name(self._platform)
 322         if not dir_name:
 323             return None
 324
 325         _log.debug('Buildbot platform dir name: "%s"', dir_name)
 326
 327         url_base = '%s/%s/' % (self._options.archive_url, dir_name)
 328         latest_revision = self._get_latest_revision(url_base)
 329         if latest_revision is None or latest_revision <= 0:
 330             return None
 331         archive_url = '%s%s/layout-test-results.zip' % (url_base, latest_revision)
 332         _log.info('  Using %s', archive_url)
 333         return archive_url
 334
 335     def _download_buildbot_archive(self):
 336         """Download layout test archive file from buildbot and return a handle to it."""
 337         url = self._get_archive_url()
 338         if url is None:
 339             return None
 340
 341         archive_file = zipfileset.ZipFileSet(url, filesystem=self._filesystem,
 342                                              zip_factory=self._zip_factory)
 343         _log.debug('Archive downloaded')
 344         return archive_file
 345
 346     def _extract_and_add_new_baselines(self, zip_file):
 347         """Extract new baselines from the zip file and add them to SVN repository.
 348
 349         Returns:
 350           List of tests that have been rebaselined or None on failure."""
 351         zip_namelist = zip_file.namelist()
 352
 353         _log.debug('zip file namelist:')
 354         for name in zip_namelist:
 355             _log.debug('  ' + name)
 356
 357         _log.debug('Platform dir: "%s"', self._platform)
 358
 359         self._rebaselined_tests = []
 360         for test_no, test in enumerate(self._rebaselining_tests):
 361             _log.debug('Test %d: %s', test_no + 1, test)
 362             self._extract_and_add_new_baseline(test, zip_file)
 363
 364     def _extract_and_add_new_baseline(self, test, zip_file):
 365         found = False
 366         scm_error = False
 367         test_basename = self._filesystem.splitext(test)[0]
 368         for suffix in BASELINE_SUFFIXES:
 369             archive_test_name = 'layout-test-results/%s-actual%s' % (test_basename, suffix)
 370             _log.debug('  Archive test file name: "%s"', archive_test_name)
 371             if not archive_test_name in zip_file.namelist():
 372                 _log.debug('  %s file not in archive.', suffix)
 373                 continue
 374
 375             found = True
 376             _log.debug('  %s file found in archive.', suffix)
 377
 378             temp_name = self._extract_from_zip_to_tempfile(zip_file, archive_test_name)
 379
 380             expected_filename = '%s-expected%s' % (test_basename, suffix)
 381             expected_fullpath = self._filesystem.join(
 382                 self._rebaseline_port.baseline_path(), expected_filename)
 383             expected_fullpath = self._filesystem.normpath(expected_fullpath)
 384             _log.debug('  Expected file full path: "%s"', expected_fullpath)
 385
 386             relpath = self._filesystem.relpath(expected_fullpath, self._target_port.layout_tests_dir())
 387
 388             # TODO(victorw): for now, the rebaselining tool checks whether
 389             # or not THIS baseline is duplicate and should be skipped.
 390             # We could improve the tool to check all baselines in upper
 391             # and lower levels and remove all duplicated baselines.
 392             if self._is_dup_baseline(temp_name, expected_fullpath, test, suffix, self._platform):
 393                 self._filesystem.remove(temp_name)
 394                 if self._filesystem.exists(expected_fullpath):
 395                     _log.info('  Removing %s' % relpath)
 396                     self._delete_baseline(expected_fullpath)
 397                 _log.debug('  %s is a duplicate' % relpath)
 398
 399                 # FIXME: We consider a duplicate baseline a success in the normal case.
 400                 # FIXME: This may not be what you want sometimes; should this be
 401                 # FIXME: controllable?
 402                 self._rebaselined_tests.append(test)
 403                 continue
 404
 405             if suffix == '.checksum' and self._png_has_same_checksum(temp_name, test, expected_fullpath):
 406                 self._filesystem.remove(temp_name)
 407                 # If an old checksum exists, delete it.
 408                 self._delete_baseline(expected_fullpath)
 409                 continue
 410
 411             self._filesystem.maybe_make_directory(self._filesystem.dirname(expected_fullpath))
 412             self._filesystem.move(temp_name, expected_fullpath)
 413
 414             path_from_base = self._filesystem.relpath(expected_fullpath)
 415             if self._scm.exists(path_from_base):
 416                 _log.info('  Updating %s' % relpath)
 417             else:
 418                 _log.info('  Adding %s' % relpath)
 419
 420             if self._scm.add(expected_fullpath, return_exit_code=True):
 421                 # FIXME: print detailed diagnose messages
 422                 scm_error = True
 423             elif suffix != '.checksum':
 424                 self._create_html_baseline_files(expected_fullpath)
 425
 426         if not found:
 427             _log.warn('No results in archive for %s' % test)
 428         elif scm_error:
 429             _log.warn('Failed to add baselines to your repository.')
 430         else:
 431             _log.debug('  Rebaseline succeeded.')
 432             self._rebaselined_tests.append(test)
 433
 434     def _extract_from_zip_to_tempfile(self, zip_file, filename):
 435         """Extracts |filename| from |zip_file|, a ZipFileSet. Returns the full
 436            path name to the extracted file."""
 437         data = zip_file.read(filename)
 438         suffix = self._filesystem.splitext(filename)[1]
 439         tempfile, temp_name = self._filesystem.open_binary_tempfile(suffix)
 440         tempfile.write(data)
 441         tempfile.close()
 442         return temp_name
 443
 444     def _png_has_same_checksum(self, checksum_path, test, checksum_expected_fullpath):
 445         """Returns True if the fallback png for |checksum_expected_fullpath|
 446         contains the same checksum."""
 447         fs = self._filesystem
 448         png_fullpath = self._first_fallback_png_for_test(test)
 449
 450         if not fs.exists(png_fullpath):
 451             _log.error('  Checksum without png file found! Expected %s to exist.' % png_fullpath)
 452             return False
 453
 454         with fs.open_binary_file_for_reading(png_fullpath) as filehandle:
 455             checksum_in_png = read_checksum_from_png.read_checksum(filehandle)
 456             checksum_in_text_file = fs.read_text_file(checksum_path)
 457             if checksum_in_png and checksum_in_png != checksum_in_text_file:
 458                 _log.error("  checksum in %s and %s don't match!  Continuing"
 459                            " to copy but please investigate." % (
 460                            checksum_expected_fullpath, png_fullpath))
 461             return checksum_in_text_file == checksum_in_png
 462
 463     def _first_fallback_png_for_test(self, test):
 464         all_baselines = self._rebaseline_port.expected_baselines(test, '.png', True)
 465         return self._filesystem.join(all_baselines[0][0], all_baselines[0][1])
 466
 467     def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix, platform):
 468         """Check whether a baseline is duplicate and can fallback to same
 469            baseline for another platform. For example, if a test has same
 470            baseline on linux and windows, then we only store windows
 471            baseline and linux baseline will fallback to the windows version.
 472
 473         Args:
 474           new_baseline: temp filename containing the new baseline results
 475           baseline_path: baseline expectation file name.
 476           test: test name.
 477           suffix: file suffix of the expected results, including dot;
 478                   e.g. '.txt' or '.png'.
 479           platform: baseline platform 'mac', 'win' or 'linux'.
 480
 481         Returns:
 482           True if the baseline is unnecessary.
 483           False otherwise.
 484         """
 485         all_baselines = self._rebaseline_port.expected_baselines(test, suffix, True)
 486
 487         for fallback_dir, fallback_file in all_baselines:
 488             if not fallback_dir or not fallback_file:
 489                 continue
 490
 491             fallback_fullpath = self._filesystem.normpath(
 492                 self._filesystem.join(fallback_dir, fallback_file))
 493             if fallback_fullpath.lower() == baseline_path.lower():
 494                 continue
 495             fallback_dir_relpath = self._filesystem.relpath(fallback_dir, self._target_port.layout_tests_dir())
 496             if fallback_dir_relpath == '':
 497                 fallback_dir_relpath = '<generic>'
 498
 499             new_output = self._filesystem.read_binary_file(new_baseline)
 500             fallback_output = self._filesystem.read_binary_file(fallback_fullpath)
 501             is_image = baseline_path.lower().endswith('.png')
 502             if not self._diff_baselines(new_output, fallback_output, is_image):
 503                 _log.info('  Skipping %s (matches %s)', test, fallback_dir_relpath)
 504                 return True
 505             return False
 506
 507         return False
 508
 509     def _diff_baselines(self, output1, output2, is_image):
 510         """Check whether two baselines are different.
 511
 512         Args:
 513           output1, output2: contents of the baselines to compare.
 514
 515         Returns:
 516           True if two files are different or have different extensions.
 517           False otherwise.
 518         """
 519
 520         if is_image:
 521             return self._port.diff_image(output1, output2)[0]
 522
 523         return self._port.compare_text(output1, output2)
 524
 525     def _delete_baseline(self, filename):
 526         """Remove the file from repository and delete it from disk.
 527
 528         Args:
 529           filename: full path of the file to delete.
 530         """
 531
 532         if not filename or not self._filesystem.isfile(filename):
 533             return
 534         self._scm.delete(filename)
 535
 536     def _create_html_baseline_files(self, baseline_fullpath):
 537         """Create baseline files (old, new and diff) in html directory.
 538
 539            The files are used to compare the rebaselining results.
 540
 541         Args:
 542           baseline_fullpath: full path of the expected baseline file.
 543         """
 544
 545         baseline_relpath = self._filesystem.relpath(baseline_fullpath)
 546         _log.debug('  Html: create baselines for "%s"', baseline_relpath)
 547
 548         if (not baseline_fullpath
 549             or not self._filesystem.exists(baseline_fullpath)):
 550             _log.debug('  Html: Does not exist: "%s"', baseline_fullpath)
 551             return
 552
 553         if not self._scm.exists(baseline_relpath):
 554             _log.debug('  Html: Does not exist in scm: "%s"', baseline_relpath)
 555             return
 556
 557         # Copy the new baseline to html directory for result comparison.
 558         baseline_filename = self._filesystem.basename(baseline_fullpath)
 559         new_file = get_result_file_fullpath(self._filesystem, self._options.html_directory,
 560                                             baseline_filename, self._platform, 'new')
 561         self._filesystem.copyfile(baseline_fullpath, new_file)
 562         _log.debug('  Html: copied new baseline file from "%s" to "%s".',
 563                   baseline_fullpath, new_file)
 564
 565         # Get the old baseline from the repository and save to the html directory.
 566         try:
 567             output = self._scm.show_head(baseline_relpath)
 568         except ScriptError, e:
 569             _log.warning(e)
 570             output = ""
 571
 572         if (not output) or (output.upper().rstrip().endswith('NO SUCH FILE OR DIRECTORY')):
 573             _log.warning('  No base file: "%s"', baseline_fullpath)
 574             return
 575         base_file = get_result_file_fullpath(self._filesystem, self._options.html_directory,
 576                                              baseline_filename, self._platform, 'old')
 577         if base_file.upper().endswith('.PNG'):
 578             self._filesystem.write_binary_file(base_file, output)
 579         else:
 580             self._filesystem.write_text_file(base_file, output)
 581         _log.debug('  Html: created old baseline file: "%s".', base_file)
 582
 583         # Get the diff between old and new baselines and save to the html dir.
 584         diff_file = get_result_file_fullpath(self._filesystem,
 585                                              self._options.html_directory,
 586                                              baseline_filename,
 587                                              self._platform, 'diff')
 588         has_diff = False
 589         if baseline_filename.upper().endswith('.TXT'):
 590             output = self._scm.diff_for_file(baseline_relpath, log=_log)
 591             if output:
 592                 self._filesystem.write_text_file(diff_file, output)
 593                 has_diff = True
 594         elif baseline_filename.upper().endswith('.PNG'):
 595             old_file = get_result_file_fullpath(self._filesystem,
 596                                                 self._options.html_directory,
 597                                                 baseline_filename,
 598                                                 self._platform, 'old')
 599             new_file = get_result_file_fullpath(self._filesystem,
 600                                                 self._options.html_directory,
 601                                                 baseline_filename,
 602                                                 self._platform, 'new')
 603             _log.debug(' Html: diffing "%s" and "%s"', old_file, new_file)
 604             old_output = self._filesystem.read_binary_file(old_file)
 605             new_output = self._filesystem.read_binary_file(new_file)
 606             image_diff = self._port.diff_image(old_output, new_output)[0]
 607             self._filesystem.write_binary_file(diff_file, image_diff)
 608
 609         if has_diff:
 610             _log.debug('  Html: created baseline diff file: "%s".', diff_file)
 611
 612
 613 class HtmlGenerator(object):
 614     """Class to generate rebaselining result comparison html."""
 615
 616     HTML_REBASELINE = ('<html>'
 617                        '<head>'
 618                        '<style>'
 619                        'body {font-family: sans-serif;}'
 620                        '.mainTable {background: #666666;}'
 621                        '.mainTable td , .mainTable th {background: white;}'
 622                        '.detail {margin-left: 10px; margin-top: 3px;}'
 623                        '</style>'
 624                        '<title>Rebaselining Result Comparison (%(time)s)'
 625                        '</title>'
 626                        '</head>'
 627                        '<body>'
 628                        '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
 629                        '%(body)s'
 630                        '</body>'
 631                        '</html>')
 632     HTML_NO_REBASELINING_TESTS = (
 633         '<p>No tests found that need rebaselining.</p>')
 634     HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
 635                        '%s</table><br>')
 636     HTML_TR_TEST = ('<tr>'
 637                     '<th style="background-color: #CDECDE; border-bottom: '
 638                     '1px solid black; font-size: 18pt; font-weight: bold" '
 639                     'colspan="5">'
 640                     '<a href="%s">%s</a>'
 641                     '</th>'
 642                     '</tr>')
 643     HTML_TEST_DETAIL = ('<div class="detail">'
 644                         '<tr>'
 645                         '<th width="100">Baseline</th>'
 646                         '<th width="100">Platform</th>'
 647                         '<th width="200">Old</th>'
 648                         '<th width="200">New</th>'
 649                         '<th width="150">Difference</th>'
 650                         '</tr>'
 651                         '%s'
 652                         '</div>')
 653     HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
 654     HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
 655     HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
 656                         '<img style="width: 200" src="%(uri)s" /></a></td>')
 657     HTML_TR = '<tr>%s</tr>'
 658
 659     def __init__(self, port, target_port, options, platforms, rebaselining_tests):
 660         self._html_directory = options.html_directory
 661         self._port = port
 662         self._target_port = target_port
 663         self._options = options
 664         self._platforms = platforms
 665         self._rebaselining_tests = rebaselining_tests
 666         self._filesystem = port._filesystem
 667         self._html_file = self._filesystem.join(options.html_directory,
 668                                                 'rebaseline.html')
 669
 670     def abspath_to_uri(self, filename):
 671         """Converts an absolute path to a file: URI."""
 672         return path.abspath_to_uri(filename, self._port._executive)
 673
 674     def generate_html(self):
 675         """Generate html file for rebaselining result comparison."""
 676
 677         _log.debug('Generating html file')
 678
 679         html_body = ''
 680         if not self._rebaselining_tests:
 681             html_body += self.HTML_NO_REBASELINING_TESTS
 682         else:
 683             tests = list(self._rebaselining_tests)
 684             tests.sort()
 685
 686             test_no = 1
 687             for test in tests:
 688                 _log.debug('Test %d: %s', test_no, test)
 689                 html_body += self._generate_html_for_one_test(test)
 690
 691         html = self.HTML_REBASELINE % ({'time': time.asctime(),
 692                                         'body': html_body})
 693         _log.debug(html)
 694
 695         self._filesystem.write_text_file(self._html_file, html)
 696         _log.debug('Baseline comparison html generated at "%s"', self._html_file)
 697
 698     def show_html(self):
 699         """Launch the rebaselining html in brwoser."""
 700
 701         _log.debug('Launching html: "%s"', self._html_file)
 702         self._port._user.open_url(self._html_file)
 703         _log.debug('Html launched.')
 704
 705     def _generate_baseline_links(self, test_basename, suffix, platform):
 706         """Generate links for baseline results (old, new and diff).
 707
 708         Args:
 709           test_basename: base filename of the test
 710           suffix: baseline file suffixes: '.txt', '.png'
 711           platform: win, linux or mac
 712
 713         Returns:
 714           html links for showing baseline results (old, new and diff)
 715         """
 716
 717         baseline_filename = '%s-expected%s' % (test_basename, suffix)
 718         _log.debug('    baseline filename: "%s"', baseline_filename)
 719
 720         new_file = get_result_file_fullpath(self._filesystem, self._html_directory,
 721                                             baseline_filename, platform, 'new')
 722         _log.debug('    New baseline file: "%s"', new_file)
 723         if not self._filesystem.exists(new_file):
 724             _log.debug('    No new baseline file: "%s"', new_file)
 725             return ''
 726
 727         old_file = get_result_file_fullpath(self._filesystem, self._html_directory,
 728                                             baseline_filename, platform, 'old')
 729         _log.debug('    Old baseline file: "%s"', old_file)
 730         if suffix == '.png':
 731             html_td_link = self.HTML_TD_LINK_IMG
 732         else:
 733             html_td_link = self.HTML_TD_LINK
 734
 735         links = ''
 736         if self._filesystem.exists(old_file):
 737             links += html_td_link % {
 738                 'uri': self.abspath_to_uri(old_file),
 739                 'name': baseline_filename}
 740         else:
 741             _log.debug('    No old baseline file: "%s"', old_file)
 742             links += self.HTML_TD_NOLINK % ''
 743
 744         links += html_td_link % {'uri': self.abspath_to_uri(new_file),
 745                                  'name': baseline_filename}
 746
 747         diff_file = get_result_file_fullpath(self._filesystem, self._html_directory,
 748                                              baseline_filename, platform, 'diff')
 749         _log.debug('    Baseline diff file: "%s"', diff_file)
 750         if self._filesystem.exists(diff_file):
 751             links += html_td_link % {'uri': self.abspath_to_uri(diff_file),
 752                                      'name': 'Diff'}
 753         else:
 754             _log.debug('    No baseline diff file: "%s"', diff_file)
 755             links += self.HTML_TD_NOLINK % ''
 756
 757         return links
 758
 759     def _generate_html_for_one_test(self, test):
 760         """Generate html for one rebaselining test.
 761
 762         Args:
 763           test: layout test name
 764
 765         Returns:
 766           html that compares baseline results for the test.
 767         """
 768
 769         test_basename = self._filesystem.basename(self._filesystem.splitext(test)[0])
 770         _log.debug('  basename: "%s"', test_basename)
 771         rows = []
 772         for suffix in BASELINE_SUFFIXES:
 773             if suffix == '.checksum':
 774                 continue
 775
 776             _log.debug('  Checking %s files', suffix)
 777             for platform in self._platforms:
 778                 links = self._generate_baseline_links(test_basename, suffix, platform)
 779                 if links:
 780                     row = self.HTML_TD_NOLINK % self._get_baseline_result_type(suffix)
 781                     row += self.HTML_TD_NOLINK % platform
 782                     row += links
 783                     _log.debug('    html row: %s', row)
 784
 785                     rows.append(self.HTML_TR % row)
 786
 787         if rows:
 788             test_path = self._filesystem.join(self._target_port.layout_tests_dir(), test)
 789             html = self.HTML_TR_TEST % (self.abspath_to_uri(test_path), test)
 790             html += self.HTML_TEST_DETAIL % ' '.join(rows)
 791
 792             _log.debug('    html for test: %s', html)
 793             return self.HTML_TABLE_TEST % html
 794
 795         return ''
 796
 797     def _get_baseline_result_type(self, suffix):
 798         """Name of the baseline result type."""
 799
 800         if suffix == '.png':
 801             return 'Pixel'
 802         elif suffix == '.txt':
 803             return 'Render Tree'
 804         else:
 805             return 'Other'
 806
 807
 808 def get_host_port_object(port_factory, options):
 809     """Return a port object for the platform we're running on."""
 810     # We want the ImageDiff logic to match that of the chromium bots, so we
 811     # force the use of a Chromium port.  We will look for either Debug or
 812     # Release versions.
 813     options.configuration = "Release"
 814     options.chromium = True
 815     port_obj = port_factory.get(options=options)
 816     if not port_obj.check_image_diff(override_step=None, logging=False):
 817         _log.debug('No release version of the image diff binary was found.')
 818         options.configuration = "Debug"
 819         port_obj = port_factory.get(options=options)
 820         if not port_obj.check_image_diff(override_step=None, logging=False):
 821             _log.error('No version of image diff was found. Check your build.')
 822             return None
 823         else:
 824             _log.debug('Found the debug version of the image diff binary.')
 825     else:
 826         _log.debug('Found the release version of the image diff binary.')
 827     return port_obj
 828
 829
 830 def parse_options(args):
 831     """Parse options and return a pair of host options and target options."""
 832     option_parser = optparse.OptionParser()
 833     option_parser.add_option('-v', '--verbose',
 834                              action='store_true',
 835                              default=False,
 836                              help='include debug-level logging.')
 837
 838     option_parser.add_option('-q', '--quiet',
 839                              action='store_true',
 840                              help='Suppress result HTML viewing')
 841
 842     option_parser.add_option('-p', '--platforms',
 843                              default=None,
 844                              help=('Comma delimited list of platforms '
 845                                    'that need rebaselining.'))
 846
 847     option_parser.add_option('-u', '--archive_url',
 848                              default=('http://build.chromium.org/f/chromium/'
 849                                       'layout_test_results'),
 850                              help=('Url to find the layout test result archive'
 851                                    ' file.'))
 852     option_parser.add_option('-U', '--force_archive_url',
 853                              help=('Url of result zip file. This option is for debugging '
 854                                    'purposes'))
 855
 856     option_parser.add_option('-b', '--backup',
 857                              action='store_true',
 858                              default=False,
 859                              help=('Whether or not to backup the original test'
 860                                    ' expectations file after rebaseline.'))
 861
 862     option_parser.add_option('-d', '--html_directory',
 863                              default='',
 864                              help=('The directory that stores the results for '
 865                                    'rebaselining comparison.'))
 866
 867     option_parser.add_option('', '--use_drt',
 868                              action='store_true',
 869                              default=False,
 870                              help=('Use ImageDiff from DumpRenderTree instead '
 871                                    'of image_diff for pixel tests.'))
 872
 873     option_parser.add_option('-w', '--webkit_canary',
 874                              action='store_true',
 875                              default=False,
 876                              help=('DEPRECATED. This flag no longer has any effect.'
 877                                    '  The canaries are always used.'))
 878
 879     option_parser.add_option('', '--target-platform',
 880                              default='chromium',
 881                              help=('The target platform to rebaseline '
 882                                    '("mac", "chromium", "qt", etc.). Defaults '
 883                                    'to "chromium".'))
 884
 885     options = option_parser.parse_args(args)[0]
 886     if options.webkit_canary:
 887         print "-w/--webkit-canary is no longer necessary, ignoring."
 888
 889     target_options = copy.copy(options)
 890     if options.target_platform == 'chromium':
 891         target_options.chromium = True
 892     options.tolerance = 0
 893
 894     return (options, target_options)
 895
 896
 897 class DebugLogHandler(logging.Handler):
 898     num_failures = 0
 899
 900     def __init__(self):
 901         logging.Handler.__init__(self)
 902         self.formatter = logging.Formatter(fmt=('%(asctime)s %(filename)s:%(lineno)-3d '
 903                                                 '%(levelname)s %(message)s'))
 904         self.setFormatter(self.formatter)
 905
 906     def emit(self, record):
 907         if record.levelno > logging.INFO:
 908             self.num_failures += 1
 909         print self.format(record)
 910
 911
 912 class NormalLogHandler(logging.Handler):
 913     last_levelno = None
 914     num_failures = 0
 915
 916     def emit(self, record):
 917         if record.levelno > logging.INFO:
 918             self.num_failures += 1
 919         if self.last_levelno != record.levelno:
 920             print
 921             self.last_levelno = record.levelno
 922         prefix = ''
 923         msg = record.getMessage()
 924         if record.levelno > logging.INFO and msg:
 925             prefix = '%s: ' % record.levelname
 926         print '%s%s' % (prefix, msg)
 927
 928
 929 def main(args):
 930     """Bootstrap function that sets up the object references we need and calls real_main()."""
 931     options, target_options = parse_options(args)
 932
 933     logger = logging.getLogger()
 934     logger.setLevel(logging.INFO)
 935     if options.verbose:
 936         log_level = logging.DEBUG
 937         log_handler = DebugLogHandler()
 938     else:
 939         log_level = logging.INFO
 940         log_handler = NormalLogHandler()
 941
 942     logger = logging.getLogger()
 943     logger.setLevel(log_level)
 944     logger.addHandler(log_handler)
 945
 946     host = Host()
 947     host._initialize_scm()
 948     target_port_obj = host.port_factory.get(None, target_options)
 949     host_port_obj = get_host_port_object(host.port_factory, options)
 950     if not host_port_obj or not target_port_obj:
 951         return 1
 952
 953     url_fetcher = urlfetcher.UrlFetcher(host_port_obj._filesystem)
 954
 955     # We use the default zip factory method.
 956     zip_factory = None
 957
 958     # FIXME: SCM module doesn't handle paths that aren't relative to the checkout_root consistently.
 959     host_port_obj._filesystem.chdir(host.scm().checkout_root)
 960
 961     ret_code = real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher, zip_factory, host.scm())
 962     if not ret_code and log_handler.num_failures:
 963         ret_code = 1
 964     print ''
 965     if ret_code:
 966         print 'Rebaselining failed.'
 967     else:
 968         print 'Rebaselining succeeded.'
 969     return ret_code
 970
 971
 972 def real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher, zip_factory, scm_obj):
 973     """Main function to produce new baselines. The Rebaseliner object uses two
 974     different Port objects - one to represent the machine the object is running
 975     on, and one to represent the port whose expectations are being updated.
 976     E.g., you can run the script on a mac and rebaseline the 'win' port.
 977
 978     Args:
 979         options: command-line argument used for the host_port_obj (see below)
 980         target_options: command_line argument used for the target_port_obj.
 981             This object may have slightly different values than |options|.
 982         host_port_obj: a Port object for the platform the script is running
 983             on. This is used to produce image and text diffs, mostly, and
 984             is usually acquired from get_host_port_obj().
 985         target_port_obj: a Port obj representing the port getting rebaselined.
 986             This is used to find the expectations file, the baseline paths,
 987             etc.
 988         url_fetcher: object used to download the build archives from the bots
 989         zip_factory: factory function used to create zip file objects for
 990             the archives.
 991         scm_obj: object used to add new baselines to the source control system.
 992     """
 993     options.html_directory = setup_html_directory(host_port_obj._filesystem, options.html_directory)
 994     all_platforms = target_port_obj.all_baseline_variants()
 995     if options.platforms:
 996         bail = False
 997         for platform in options.platforms:
 998             if not platform in all_platforms:
 999                 _log.error('Invalid platform: "%s"' % (platform))
1000                 bail = True
1001         if bail:
1002             return 1
1003         rebaseline_platforms = options.platforms
1004     else:
1005         rebaseline_platforms = all_platforms
1006
1007     # FIXME: These log messages will be wrong if ports store baselines outside
1008     # of layout_tests_dir(), but the code should work correctly.
1009     layout_tests_dir = target_port_obj.layout_tests_dir()
1010     expectations_path = target_port_obj.path_to_test_expectations_file()
1011     _log.info('Using %s' % layout_tests_dir)
1012     _log.info('  and %s' % expectations_path)
1013
1014     rebaselined_tests = set()
1015     logged_before = False
1016     for platform in rebaseline_platforms:
1017         rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
1018                                   platform, options, url_fetcher, zip_factory,
1019                                   scm_obj, logged_before)
1020
1021         _log.debug('')
1022         log_dashed_string('Rebaseline started', platform)
1023         if rebaseliner.run():
1024             log_dashed_string('Rebaseline done', platform)
1025         else:
1026             log_dashed_string('Rebaseline failed', platform)
1027
1028         rebaselined_tests |= set(rebaseliner.get_rebaselined_tests())
1029         logged_before = rebaseliner.did_log
1030
1031     if rebaselined_tests:
1032         rebaseliner.remove_rebaselining_expectations(rebaselined_tests,
1033                                                      options.backup)
1034
1035     _log.debug('')
1036     log_dashed_string('Rebaselining result comparison started')
1037     html_generator = HtmlGenerator(host_port_obj,
1038                                    target_port_obj,
1039                                    options,
1040                                    rebaseline_platforms,
1041                                    rebaselined_tests)
1042     html_generator.generate_html()
1043     if not options.quiet:
1044         html_generator.show_html()
1045     log_dashed_string('Rebaselining result comparison done')
1046
1047     return 0
1048
1049
1050 if '__main__' == __name__:
1051     sys.exit(main(sys.argv[1:]))