src/third_party/WebKit/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py

   1 # Copyright (c) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import fnmatch
  30 import os
  31 import os.path
  32 import BaseHTTPServer
  33
  34 from webkitpy.layout_tests.port.base import Port
  35 from webkitpy.tool.servers.reflectionhandler import ReflectionHandler
  36
  37
  38 STATE_NEEDS_REBASELINE = 'needs_rebaseline'
  39 STATE_REBASELINE_FAILED = 'rebaseline_failed'
  40 STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded'
  41
  42
  43 def _get_actual_result_files(test_file, test_config):
  44     test_name, _ = os.path.splitext(test_file)
  45     test_directory = os.path.dirname(test_file)
  46
  47     test_results_directory = test_config.filesystem.join(
  48         test_config.results_directory, test_directory)
  49     actual_pattern = os.path.basename(test_name) + '-actual.*'
  50     actual_files = []
  51     for filename in test_config.filesystem.listdir(test_results_directory):
  52         if fnmatch.fnmatch(filename, actual_pattern):
  53             actual_files.append(filename)
  54     actual_files.sort()
  55     return tuple(actual_files)
  56
  57
  58 def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log):
  59     test_name, _ = os.path.splitext(test_file)
  60     test_directory = os.path.dirname(test_name)
  61
  62     log('Rebaselining %s...' % test_name)
  63
  64     actual_result_files = _get_actual_result_files(test_file, test_config)
  65     filesystem = test_config.filesystem
  66     scm = test_config.scm
  67     layout_tests_directory = test_config.layout_tests_directory
  68     results_directory = test_config.results_directory
  69     target_expectations_directory = filesystem.join(
  70         layout_tests_directory, 'platform', baseline_target, test_directory)
  71     test_results_directory = test_config.filesystem.join(
  72         test_config.results_directory, test_directory)
  73
  74     # If requested, move current baselines out
  75     current_baselines = get_test_baselines(test_file, test_config)
  76     if baseline_target in current_baselines and baseline_move_to != 'none':
  77         log('  Moving current %s baselines to %s' %
  78             (baseline_target, baseline_move_to))
  79
  80         # See which ones we need to move (only those that are about to be
  81         # updated), and make sure we're not clobbering any files in the
  82         # destination.
  83         current_extensions = set(current_baselines[baseline_target].keys())
  84         actual_result_extensions = [
  85             os.path.splitext(f)[1] for f in actual_result_files]
  86         extensions_to_move = current_extensions.intersection(
  87             actual_result_extensions)
  88
  89         if extensions_to_move.intersection(
  90             current_baselines.get(baseline_move_to, {}).keys()):
  91             log('    Already had baselines in %s, could not move existing '
  92                 '%s ones' % (baseline_move_to, baseline_target))
  93             return False
  94
  95         # Do the actual move.
  96         if extensions_to_move:
  97             if not _move_test_baselines(
  98                 test_file,
  99                 list(extensions_to_move),
 100                 baseline_target,
 101                 baseline_move_to,
 102                 test_config,
 103                 log):
 104                 return False
 105         else:
 106             log('    No current baselines to move')
 107
 108     log('  Updating baselines for %s' % baseline_target)
 109     filesystem.maybe_make_directory(target_expectations_directory)
 110     for source_file in actual_result_files:
 111         source_path = filesystem.join(test_results_directory, source_file)
 112         destination_file = source_file.replace('-actual', '-expected')
 113         destination_path = filesystem.join(
 114             target_expectations_directory, destination_file)
 115         filesystem.copyfile(source_path, destination_path)
 116         exit_code = scm.add(destination_path, return_exit_code=True)
 117         if exit_code:
 118             log('    Could not update %s in SCM, exit code %d' %
 119                 (destination_file, exit_code))
 120             return False
 121         else:
 122             log('    Updated %s' % destination_file)
 123
 124     return True
 125
 126
 127 def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log):
 128     test_file_name = os.path.splitext(os.path.basename(test_file))[0]
 129     test_directory = os.path.dirname(test_file)
 130     filesystem = test_config.filesystem
 131
 132     # Want predictable output order for unit tests.
 133     extensions_to_move.sort()
 134
 135     source_directory = os.path.join(
 136         test_config.layout_tests_directory,
 137         'platform',
 138         source_platform,
 139         test_directory)
 140     destination_directory = os.path.join(
 141         test_config.layout_tests_directory,
 142         'platform',
 143         destination_platform,
 144         test_directory)
 145     filesystem.maybe_make_directory(destination_directory)
 146
 147     for extension in extensions_to_move:
 148         file_name = test_file_name + '-expected' + extension
 149         source_path = filesystem.join(source_directory, file_name)
 150         destination_path = filesystem.join(destination_directory, file_name)
 151         filesystem.copyfile(source_path, destination_path)
 152         exit_code = test_config.scm.add(destination_path, return_exit_code=True)
 153         if exit_code:
 154             log('    Could not update %s in SCM, exit code %d' %
 155                 (file_name, exit_code))
 156             return False
 157         else:
 158             log('    Moved %s' % file_name)
 159
 160     return True
 161
 162
 163 def get_test_baselines(test_file, test_config):
 164     # FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic.
 165     class AllPlatformsPort(Port):
 166         def __init__(self, host):
 167             super(AllPlatformsPort, self).__init__(host, 'mac')
 168             self._platforms_by_directory = dict([(self._webkit_baseline_path(p), p) for p in test_config.platforms])
 169
 170         def baseline_search_path(self):
 171             return self._platforms_by_directory.keys()
 172
 173         def platform_from_directory(self, directory):
 174             return self._platforms_by_directory[directory]
 175
 176     test_path = test_config.filesystem.join(test_config.layout_tests_directory, test_file)
 177     host = test_config.host
 178     all_platforms_port = AllPlatformsPort(host)
 179
 180     all_test_baselines = {}
 181     for baseline_extension in ('.txt', '.checksum', '.png'):
 182         test_baselines = test_config.test_port.expected_baselines(test_file, baseline_extension)
 183         baselines = all_platforms_port.expected_baselines(test_file, baseline_extension, all_baselines=True)
 184         for platform_directory, expected_filename in baselines:
 185             if not platform_directory:
 186                 continue
 187             if platform_directory == test_config.layout_tests_directory:
 188                 platform = 'base'
 189             else:
 190                 platform = all_platforms_port.platform_from_directory(platform_directory)
 191             platform_baselines = all_test_baselines.setdefault(platform, {})
 192             was_used_for_test = (platform_directory, expected_filename) in test_baselines
 193             platform_baselines[baseline_extension] = was_used_for_test
 194
 195     return all_test_baselines
 196
 197
 198 class RebaselineHTTPServer(BaseHTTPServer.HTTPServer):
 199     def __init__(self, httpd_port, config):
 200         server_name = ""
 201         BaseHTTPServer.HTTPServer.__init__(self, (server_name, httpd_port), RebaselineHTTPRequestHandler)
 202         self.test_config = config['test_config']
 203         self.results_json = config['results_json']
 204         self.platforms_json = config['platforms_json']
 205
 206
 207 class RebaselineHTTPRequestHandler(ReflectionHandler):
 208     STATIC_FILE_NAMES = frozenset([
 209         "index.html",
 210         "loupe.js",
 211         "main.js",
 212         "main.css",
 213         "queue.js",
 214         "util.js",
 215     ])
 216
 217     STATIC_FILE_DIRECTORY = os.path.join(os.path.dirname(__file__), "data", "rebaselineserver")
 218
 219     def results_json(self):
 220         self._serve_json(self.server.results_json)
 221
 222     def test_config(self):
 223         self._serve_json(self.server.test_config)
 224
 225     def platforms_json(self):
 226         self._serve_json(self.server.platforms_json)
 227
 228     def rebaseline(self):
 229         test = self.query['test'][0]
 230         baseline_target = self.query['baseline-target'][0]
 231         baseline_move_to = self.query['baseline-move-to'][0]
 232         test_json = self.server.results_json['tests'][test]
 233
 234         if test_json['state'] != STATE_NEEDS_REBASELINE:
 235             self.send_error(400, "Test %s is in unexpected state: %s" % (test, test_json["state"]))
 236             return
 237
 238         log = []
 239         success = _rebaseline_test(
 240             test,
 241             baseline_target,
 242             baseline_move_to,
 243             self.server.test_config,
 244             log=lambda l: log.append(l))
 245
 246         if success:
 247             test_json['state'] = STATE_REBASELINE_SUCCEEDED
 248             self.send_response(200)
 249         else:
 250             test_json['state'] = STATE_REBASELINE_FAILED
 251             self.send_response(500)
 252
 253         self.send_header('Content-type', 'text/plain')
 254         self.end_headers()
 255         self.wfile.write('\n'.join(log))
 256
 257     def test_result(self):
 258         test_name, _ = os.path.splitext(self.query['test'][0])
 259         mode = self.query['mode'][0]
 260         if mode == 'expected-image':
 261             file_name = test_name + '-expected.png'
 262         elif mode == 'actual-image':
 263             file_name = test_name + '-actual.png'
 264         if mode == 'expected-checksum':
 265             file_name = test_name + '-expected.checksum'
 266         elif mode == 'actual-checksum':
 267             file_name = test_name + '-actual.checksum'
 268         elif mode == 'diff-image':
 269             file_name = test_name + '-diff.png'
 270         if mode == 'expected-text':
 271             file_name = test_name + '-expected.txt'
 272         elif mode == 'actual-text':
 273             file_name = test_name + '-actual.txt'
 274         elif mode == 'diff-text':
 275             file_name = test_name + '-diff.txt'
 276         elif mode == 'diff-text-pretty':
 277             file_name = test_name + '-pretty-diff.html'
 278
 279         file_path = os.path.join(self.server.test_config.results_directory, file_name)
 280
 281         # Let results be cached for 60 seconds, so that they can be pre-fetched
 282         # by the UI
 283         self._serve_file(file_path, cacheable_seconds=60)