src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py

   1 # Copyright (C) 2011 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29
  30 import logging
  31
  32 from webkitpy.layout_tests.controllers import repaint_overlay
  33 from webkitpy.layout_tests.models import test_failures
  34
  35
  36 _log = logging.getLogger(__name__)
  37
  38
  39 def write_test_result(filesystem, port, results_directory, test_name, driver_output,
  40                       expected_driver_output, failures):
  41     """Write the test result to the result output directory."""
  42     root_output_dir = results_directory
  43     writer = TestResultWriter(filesystem, port, root_output_dir, test_name)
  44
  45     if driver_output.error:
  46         writer.write_stderr(driver_output.error)
  47
  48     for failure in failures:
  49         # FIXME: Instead of this long 'if' block, each failure class might
  50         # have a responsibility for writing a test result.
  51         if isinstance(failure, (test_failures.FailureMissingResult,
  52                                 test_failures.FailureTextMismatch,
  53                                 test_failures.FailureTestHarnessAssertion)):
  54             writer.write_text_files(driver_output.text, expected_driver_output.text)
  55             writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
  56         elif isinstance(failure, test_failures.FailureMissingImage):
  57             writer.write_image_files(driver_output.image, expected_image=None)
  58         elif isinstance(failure, test_failures.FailureMissingImageHash):
  59             writer.write_image_files(driver_output.image, expected_driver_output.image)
  60         elif isinstance(failure, test_failures.FailureImageHashMismatch):
  61             writer.write_image_files(driver_output.image, expected_driver_output.image)
  62             writer.write_image_diff_files(driver_output.image_diff)
  63         elif isinstance(failure, (test_failures.FailureAudioMismatch,
  64                                   test_failures.FailureMissingAudio)):
  65             writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
  66         elif isinstance(failure, test_failures.FailureCrash):
  67             crashed_driver_output = expected_driver_output if failure.is_reftest else driver_output
  68             writer.write_crash_log(crashed_driver_output.crash_log)
  69         elif isinstance(failure, test_failures.FailureLeak):
  70             writer.write_leak_log(driver_output.leak_log)
  71         elif isinstance(failure, test_failures.FailureReftestMismatch):
  72             writer.write_image_files(driver_output.image, expected_driver_output.image)
  73             # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
  74             # FIXME: We should always have 2 images here.
  75             if driver_output.image and expected_driver_output.image:
  76                 diff_image, err_str = port.diff_image(expected_driver_output.image, driver_output.image)
  77                 if diff_image:
  78                     writer.write_image_diff_files(diff_image)
  79                 else:
  80                     _log.warn('ref test mismatch did not produce an image diff.')
  81             writer.write_image_files(driver_output.image, expected_image=None)
  82             if filesystem.exists(failure.reference_filename):
  83                 writer.write_reftest(failure.reference_filename)
  84             else:
  85                 _log.warn("reference %s was not found" % failure.reference_filename)
  86         elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
  87             writer.write_image_files(driver_output.image, expected_image=None)
  88             if filesystem.exists(failure.reference_filename):
  89                 writer.write_reftest(failure.reference_filename)
  90             else:
  91                 _log.warn("reference %s was not found" % failure.reference_filename)
  92         else:
  93             assert isinstance(failure, (test_failures.FailureTimeout, test_failures.FailureReftestNoImagesGenerated))
  94
  95         writer.create_repaint_overlay_result(driver_output.text, expected_driver_output.text)
  96
  97
  98 class TestResultWriter(object):
  99     """A class which handles all writing operations to the result directory."""
 100
 101     # Filename pieces when writing failures to the test results directory.
 102     FILENAME_SUFFIX_ACTUAL = "-actual"
 103     FILENAME_SUFFIX_EXPECTED = "-expected"
 104     FILENAME_SUFFIX_DIFF = "-diff"
 105     FILENAME_SUFFIX_STDERR = "-stderr"
 106     FILENAME_SUFFIX_CRASH_LOG = "-crash-log"
 107     FILENAME_SUFFIX_SAMPLE = "-sample"
 108     FILENAME_SUFFIX_LEAK_LOG = "-leak-log"
 109     FILENAME_SUFFIX_WDIFF = "-wdiff.html"
 110     FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html"
 111     FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
 112     FILENAME_SUFFIX_IMAGE_DIFFS_HTML = "-diffs.html"
 113     FILENAME_SUFFIX_OVERLAY = "-overlay.html"
 114
 115     def __init__(self, filesystem, port, root_output_dir, test_name):
 116         self._filesystem = filesystem
 117         self._port = port
 118         self._root_output_dir = root_output_dir
 119         self._test_name = test_name
 120
 121     def _make_output_directory(self):
 122         """Creates the output directory (if needed) for a given test filename."""
 123         fs = self._filesystem
 124         output_filename = fs.join(self._root_output_dir, self._test_name)
 125         fs.maybe_make_directory(fs.dirname(output_filename))
 126
 127     def output_filename(self, modifier):
 128         """Returns a filename inside the output dir that contains modifier.
 129
 130         For example, if test name is "fast/dom/foo.html" and modifier is "-expected.txt",
 131         the return value is "/<path-to-root-output-dir>/fast/dom/foo-expected.txt".
 132
 133         Args:
 134           modifier: a string to replace the extension of filename with
 135
 136         Return:
 137           The absolute path to the output filename
 138         """
 139         fs = self._filesystem
 140         output_filename = fs.join(self._root_output_dir, self._test_name)
 141         return fs.splitext(output_filename)[0] + modifier
 142
 143     def _write_file(self, path, contents):
 144         if contents is not None:
 145             self._make_output_directory()
 146             self._filesystem.write_binary_file(path, contents)
 147
 148     def _output_testname(self, modifier):
 149         fs = self._filesystem
 150         return fs.splitext(fs.basename(self._test_name))[0] + modifier
 151
 152     def write_output_files(self, file_type, output, expected):
 153         """Writes the test output, the expected output in the results directory.
 154
 155         The full output filename of the actual, for example, will be
 156           <filename>-actual<file_type>
 157         For instance,
 158           my_test-actual.txt
 159
 160         Args:
 161           file_type: A string describing the test output file type, e.g. ".txt"
 162           output: A string containing the test output
 163           expected: A string containing the expected test output
 164         """
 165         actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
 166         expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
 167
 168         self._write_file(actual_filename, output)
 169         self._write_file(expected_filename, expected)
 170
 171     def write_stderr(self, error):
 172         filename = self.output_filename(self.FILENAME_SUFFIX_STDERR + ".txt")
 173         self._write_file(filename, error)
 174
 175     def write_crash_log(self, crash_log):
 176         filename = self.output_filename(self.FILENAME_SUFFIX_CRASH_LOG + ".txt")
 177         self._write_file(filename, crash_log.encode('utf8', 'replace'))
 178
 179     def write_leak_log(self, leak_log):
 180         filename = self.output_filename(self.FILENAME_SUFFIX_LEAK_LOG + ".txt")
 181         self._write_file(filename, leak_log)
 182
 183     def copy_sample_file(self, sample_file):
 184         filename = self.output_filename(self.FILENAME_SUFFIX_SAMPLE + ".txt")
 185         self._filesystem.copyfile(sample_file, filename)
 186
 187     def write_text_files(self, actual_text, expected_text):
 188         self.write_output_files(".txt", actual_text, expected_text)
 189
 190     def create_text_diff_and_write_result(self, actual_text, expected_text):
 191         # FIXME: This function is actually doing the diffs as well as writing results.
 192         # It might be better to extract code which does 'diff' and make it a separate function.
 193         if not actual_text or not expected_text:
 194             return
 195
 196         file_type = '.txt'
 197         actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
 198         expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
 199         # We treat diff output as binary. Diff output may contain multiple files
 200         # in conflicting encodings.
 201         diff = self._port.diff_text(expected_text, actual_text, expected_filename, actual_filename)
 202         diff_filename = self.output_filename(self.FILENAME_SUFFIX_DIFF + file_type)
 203         self._write_file(diff_filename, diff)
 204
 205         # Shell out to wdiff to get colored inline diffs.
 206         if self._port.wdiff_available():
 207             wdiff = self._port.wdiff_text(expected_filename, actual_filename)
 208             wdiff_filename = self.output_filename(self.FILENAME_SUFFIX_WDIFF)
 209             self._write_file(wdiff_filename, wdiff)
 210
 211         # Use WebKit's PrettyPatch.rb to get an HTML diff.
 212         if self._port.pretty_patch_available():
 213             pretty_patch = self._port.pretty_patch_text(diff_filename)
 214             pretty_patch_filename = self.output_filename(self.FILENAME_SUFFIX_PRETTY_PATCH)
 215             self._write_file(pretty_patch_filename, pretty_patch)
 216
 217     def create_repaint_overlay_result(self, actual_text, expected_text):
 218         html = repaint_overlay.generate_repaint_overlay_html(self._test_name, actual_text, expected_text)
 219         if html:
 220             overlay_filename = self.output_filename(self.FILENAME_SUFFIX_OVERLAY)
 221             self._write_file(overlay_filename, html)
 222
 223     def write_audio_files(self, actual_audio, expected_audio):
 224         self.write_output_files('.wav', actual_audio, expected_audio)
 225
 226     def write_image_files(self, actual_image, expected_image):
 227         self.write_output_files('.png', actual_image, expected_image)
 228
 229     def write_image_diff_files(self, image_diff):
 230         diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
 231         self._write_file(diff_filename, image_diff)
 232
 233         diffs_html_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFFS_HTML)
 234         # FIXME: old-run-webkit-tests shows the diff percentage as the text contents of the "diff" link.
 235         # FIXME: old-run-webkit-tests include a link to the test file.
 236         html = """<!DOCTYPE HTML>
 237 <html>
 238 <head>
 239 <title>%(title)s</title>
 240 <style>.label{font-weight:bold}</style>
 241 </head>
 242 <body>
 243 Difference between images: <a href="%(diff_filename)s">diff</a><br>
 244 <div class=imageText></div>
 245 <div class=imageContainer data-prefix="%(prefix)s">Loading...</div>
 246 <script>
 247 (function() {
 248     var preloadedImageCount = 0;
 249     function preloadComplete() {
 250         ++preloadedImageCount;
 251         if (preloadedImageCount < 2)
 252             return;
 253         toggleImages();
 254         setInterval(toggleImages, 2000)
 255     }
 256
 257     function preloadImage(url) {
 258         image = new Image();
 259         image.addEventListener('load', preloadComplete);
 260         image.src = url;
 261         return image;
 262     }
 263
 264     function toggleImages() {
 265         if (text.textContent == 'Expected Image') {
 266             text.textContent = 'Actual Image';
 267             container.replaceChild(actualImage, container.firstChild);
 268         } else {
 269             text.textContent = 'Expected Image';
 270             container.replaceChild(expectedImage, container.firstChild);
 271         }
 272     }
 273
 274     var text = document.querySelector('.imageText');
 275     var container = document.querySelector('.imageContainer');
 276     var actualImage = preloadImage(container.getAttribute('data-prefix') + '-actual.png');
 277     var expectedImage = preloadImage(container.getAttribute('data-prefix') + '-expected.png');
 278 })();
 279 </script>
 280 </body>
 281 </html>
 282 """ % {
 283             'title': self._test_name,
 284             'diff_filename': self._output_testname(self.FILENAME_SUFFIX_IMAGE_DIFF),
 285             'prefix': self._output_testname(''),
 286         }
 287         self._write_file(diffs_html_filename, html)
 288
 289     def write_reftest(self, src_filepath):
 290         fs = self._filesystem
 291         dst_dir = fs.dirname(fs.join(self._root_output_dir, self._test_name))
 292         dst_filepath = fs.join(dst_dir, fs.basename(src_filepath))
 293         self._write_file(dst_filepath, fs.read_binary_file(src_filepath))