Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

   1 # Copyright (C) 2011 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29
  30 import logging
  31 import re
  32 import time
  33
  34 from webkitpy.layout_tests.controllers import test_result_writer
  35 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput
  36 from webkitpy.layout_tests.models import test_expectations
  37 from webkitpy.layout_tests.models import test_failures
  38 from webkitpy.layout_tests.models.test_results import TestResult
  39
  40
  41 _log = logging.getLogger(__name__)
  42
  43
  44 def run_single_test(port, options, test_input, driver, worker_name):
  45     runner = SingleTestRunner(options, port, driver, test_input, worker_name)
  46     return runner.run()
  47
  48
  49 class SingleTestRunner:
  50
  51     def __init__(self, options, port, driver, test_input, worker_name):
  52         self._options = options
  53         self._port = port
  54         self._driver = driver
  55         self._timeout = test_input.timeout
  56         self._worker_name = worker_name
  57         self._test_name = test_input.test_name
  58
  59         self._is_reftest = False
  60         self._is_mismatch_reftest = False
  61         self._reference_filename = None
  62
  63         fs = port._filesystem
  64         if test_input.ref_file:
  65             self._is_reftest = True
  66             self._reference_filename = fs.join(self._port.layout_tests_dir(), test_input.ref_file)
  67             self._is_mismatch_reftest = test_input.is_mismatch_reftest
  68             return
  69
  70         reftest_expected_filename = port.reftest_expected_filename(self._test_name)
  71         if fs.exists(reftest_expected_filename):
  72             self._is_reftest = True
  73             self._reference_filename = reftest_expected_filename
  74
  75         reftest_expected_mismatch_filename = port.reftest_expected_mismatch_filename(self._test_name)
  76         if fs.exists(reftest_expected_mismatch_filename):
  77             if self._is_reftest:
  78                 _log.error('It is not allowed that one test file has both'
  79                            ' expected.html file and expected-mismatch.html file'
  80                            ' at the same time. Please remove either %s or %s.',
  81                            reftest_expected_filename, reftest_expected_mismatch_filename)
  82             else:
  83                 self._is_reftest = True
  84                 self._is_mismatch_reftest = True
  85                 self._reference_filename = reftest_expected_mismatch_filename
  86
  87         if self._is_reftest:
  88             # Detect and report a test which has a wrong combination of expectation files.
  89             # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
  90             # 'foo-expected.txt', we should warn users. One test file must be used exclusively
  91             # in either layout tests or reftests, but not in both.
  92             for suffix in ('.txt', '.png', '.wav'):
  93                 expected_filename = self._port.expected_filename(self._test_name, suffix)
  94                 if fs.exists(expected_filename):
  95                     _log.error('The reftest (%s) can not have an expectation file (%s).'
  96                                ' Please remove that file.', self._test_name, expected_filename)
  97
  98     def _expected_driver_output(self):
  99         return DriverOutput(self._port.expected_text(self._test_name),
 100                                  self._port.expected_image(self._test_name),
 101                                  self._port.expected_checksum(self._test_name),
 102                                  self._port.expected_audio(self._test_name))
 103
 104     def _should_fetch_expected_checksum(self):
 105         return (self._options.pixel_tests and
 106                 not (self._options.new_baseline or self._options.reset_results))
 107
 108     def _driver_input(self):
 109         # The image hash is used to avoid doing an image dump if the
 110         # checksums match, so it should be set to a blank value if we
 111         # are generating a new baseline.  (Otherwise, an image from a
 112         # previous run will be copied into the baseline."""
 113         image_hash = None
 114         if self._should_fetch_expected_checksum():
 115             image_hash = self._port.expected_checksum(self._test_name)
 116         return DriverInput(self._test_name, self._timeout, image_hash, self._is_reftest)
 117
 118     def run(self):
 119         if self._is_reftest:
 120             if self._port.get_option('no_ref_tests') or self._options.new_baseline or self._options.reset_results:
 121                 result = TestResult(self._test_name)
 122                 result.type = test_expectations.SKIP
 123                 return result
 124             return self._run_reftest()
 125         if self._options.new_baseline or self._options.reset_results:
 126             return self._run_rebaseline()
 127         return self._run_compare_test()
 128
 129     def _run_compare_test(self):
 130         driver_output = self._driver.run_test(self._driver_input())
 131         expected_driver_output = self._expected_driver_output()
 132         test_result = self._compare_output(driver_output, expected_driver_output)
 133         if self._options.new_test_results:
 134             self._add_missing_baselines(test_result, driver_output)
 135         test_result_writer.write_test_result(self._port, self._test_name, driver_output, expected_driver_output, test_result.failures)
 136         return test_result
 137
 138     def _run_rebaseline(self):
 139         driver_output = self._driver.run_test(self._driver_input())
 140         failures = self._handle_error(driver_output)
 141         test_result_writer.write_test_result(self._port, self._test_name, driver_output, None, failures)
 142         # FIXME: It the test crashed or timed out, it might be bettter to avoid
 143         # to write new baselines.
 144         self._overwrite_baselines(driver_output)
 145         return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr())
 146
 147     _render_tree_dump_pattern = re.compile(r"^layer at \(\d+,\d+\) size \d+x\d+\n")
 148
 149     def _add_missing_baselines(self, test_result, driver_output):
 150         missingImage = test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash)
 151         if test_result.has_failure_matching_types(test_failures.FailureMissingResult):
 152             self._save_baseline_data(driver_output.text, ".txt", SingleTestRunner._render_tree_dump_pattern.match(driver_output.text))
 153         if test_result.has_failure_matching_types(test_failures.FailureMissingAudio):
 154             self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=False)
 155         if missingImage:
 156             self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=True)
 157
 158     def _overwrite_baselines(self, driver_output):
 159         # Although all DumpRenderTree output should be utf-8,
 160         # we do not ever decode it inside run-webkit-tests.  For some tests
 161         # DumpRenderTree may not output utf-8 text (e.g. webarchives).
 162         self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=self._options.new_baseline)
 163         self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=self._options.new_baseline)
 164         if self._options.pixel_tests:
 165             self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=self._options.new_baseline)
 166
 167     def _save_baseline_data(self, data, modifier, generate_new_baseline=True):
 168         """Saves a new baseline file into the port's baseline directory.
 169
 170         The file will be named simply "<test>-expected<modifier>", suitable for
 171         use as the expected results in a later run.
 172
 173         Args:
 174           data: result to be saved as the new baseline
 175           modifier: type of the result file, e.g. ".txt" or ".png"
 176           generate_new_baseline: whether to enerate a new, platform-specific
 177             baseline, or update the existing one
 178         """
 179         if data is None:
 180             return
 181         port = self._port
 182         fs = port._filesystem
 183         if generate_new_baseline:
 184             relative_dir = fs.dirname(self._test_name)
 185             baseline_path = port.baseline_path()
 186             output_dir = fs.join(baseline_path, relative_dir)
 187             output_file = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + modifier)
 188             fs.maybe_make_directory(output_dir)
 189             output_path = fs.join(output_dir, output_file)
 190         else:
 191             output_path = port.expected_filename(self._test_name, modifier)
 192
 193         result_name = fs.relpath(output_path, port.layout_tests_dir())
 194         _log.info('Writing new expected result "%s"' % result_name)
 195         port.update_baseline(output_path, data)
 196
 197     def _handle_error(self, driver_output, reference_filename=None):
 198         """Returns test failures if some unusual errors happen in driver's run.
 199
 200         Args:
 201           driver_output: The output from the driver.
 202           reference_filename: The full path to the reference file which produced the driver_output.
 203               This arg is optional and should be used only in reftests until we have a better way to know
 204               which html file is used for producing the driver_output.
 205         """
 206         failures = []
 207         fs = self._port._filesystem
 208         if driver_output.timeout:
 209             failures.append(test_failures.FailureTimeout(bool(reference_filename)))
 210
 211         if reference_filename:
 212             testname = self._port.relative_test_filename(reference_filename)
 213         else:
 214             testname = self._test_name
 215
 216         if driver_output.crash:
 217             failures.append(test_failures.FailureCrash(bool(reference_filename)))
 218             _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, testname,
 219                                                       driver_output.error))
 220         elif driver_output.error:
 221             _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, testname,
 222                                                            driver_output.error))
 223         return failures
 224
 225     def _compare_output(self, driver_output, expected_driver_output):
 226         failures = []
 227         failures.extend(self._handle_error(driver_output))
 228
 229         if driver_output.crash:
 230             # Don't continue any more if we already have a crash.
 231             # In case of timeouts, we continue since we still want to see the text and image output.
 232             return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr())
 233
 234         failures.extend(self._compare_text(driver_output.text, expected_driver_output.text))
 235         failures.extend(self._compare_audio(driver_output.audio, expected_driver_output.audio))
 236         if self._options.pixel_tests:
 237             failures.extend(self._compare_image(driver_output, expected_driver_output))
 238         return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr())
 239
 240     def _compare_text(self, actual_text, expected_text):
 241         failures = []
 242         if (expected_text and actual_text and
 243             # Assuming expected_text is already normalized.
 244             self._port.compare_text(self._get_normalized_output_text(actual_text), expected_text)):
 245             failures.append(test_failures.FailureTextMismatch())
 246         elif actual_text and not expected_text:
 247             failures.append(test_failures.FailureMissingResult())
 248         return failures
 249
 250     def _compare_audio(self, actual_audio, expected_audio):
 251         failures = []
 252         if (expected_audio and actual_audio and
 253             self._port.compare_audio(actual_audio, expected_audio)):
 254             failures.append(test_failures.FailureAudioMismatch())
 255         elif actual_audio and not expected_audio:
 256             failures.append(test_failures.FailureMissingAudio())
 257         return failures
 258
 259     def _get_normalized_output_text(self, output):
 260         """Returns the normalized text output, i.e. the output in which
 261         the end-of-line characters are normalized to "\n"."""
 262         # Running tests on Windows produces "\r\n".  The "\n" part is helpfully
 263         # changed to "\r\n" by our system (Python/Cygwin), resulting in
 264         # "\r\r\n", when, in fact, we wanted to compare the text output with
 265         # the normalized text expectation files.
 266         return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n")
 267
 268     # FIXME: This function also creates the image diff. Maybe that work should
 269     # be handled elsewhere?
 270     def _compare_image(self, driver_output, expected_driver_output):
 271         failures = []
 272         # If we didn't produce a hash file, this test must be text-only.
 273         if driver_output.image_hash is None:
 274             return failures
 275         if not expected_driver_output.image:
 276             failures.append(test_failures.FailureMissingImage())
 277         elif not expected_driver_output.image_hash:
 278             failures.append(test_failures.FailureMissingImageHash())
 279         elif driver_output.image_hash != expected_driver_output.image_hash:
 280             diff_result = self._port.diff_image(driver_output.image, expected_driver_output.image)
 281             driver_output.image_diff = diff_result[0]
 282             if driver_output.image_diff:
 283                 failures.append(test_failures.FailureImageHashMismatch(diff_result[1]))
 284         return failures
 285
 286     def _run_reftest(self):
 287         driver_output1 = self._driver.run_test(self._driver_input())
 288         reference_test_name = self._port.relative_test_filename(self._reference_filename)
 289         driver_output2 = self._driver.run_test(DriverInput(reference_test_name, self._timeout, driver_output1.image_hash, self._is_reftest))
 290         test_result = self._compare_output_with_reference(driver_output1, driver_output2)
 291
 292         test_result_writer.write_test_result(self._port, self._test_name, driver_output1, driver_output2, test_result.failures)
 293         return test_result
 294
 295     def _compare_output_with_reference(self, driver_output1, driver_output2):
 296         total_test_time = driver_output1.test_time + driver_output2.test_time
 297         has_stderr = driver_output1.has_stderr() or driver_output2.has_stderr()
 298         failures = []
 299         failures.extend(self._handle_error(driver_output1))
 300         if failures:
 301             # Don't continue any more if we already have crash or timeout.
 302             return TestResult(self._test_name, failures, total_test_time, has_stderr)
 303         failures.extend(self._handle_error(driver_output2, reference_filename=self._reference_filename))
 304         if failures:
 305             return TestResult(self._test_name, failures, total_test_time, has_stderr)
 306
 307         assert(driver_output1.image_hash or driver_output2.image_hash)
 308
 309         if self._is_mismatch_reftest:
 310             if driver_output1.image_hash == driver_output2.image_hash:
 311                 failures.append(test_failures.FailureReftestMismatchDidNotOccur(self._reference_filename))
 312         elif driver_output1.image_hash != driver_output2.image_hash:
 313             failures.append(test_failures.FailureReftestMismatch(self._reference_filename))
 314         return TestResult(self._test_name, failures, total_test_time, has_stderr)