src/third_party/WebKit/Tools/Scripts/webkitpy/tool/servers/rebaselineserver_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import json
  30 import unittest
  31
  32 from webkitpy.common.net import layouttestresults_unittest
  33 from webkitpy.common.host_mock import MockHost
  34 from webkitpy.layout_tests.layout_package.json_results_generator import strip_json_wrapper
  35 from webkitpy.layout_tests.port.base import Port
  36 from webkitpy.tool.commands.rebaselineserver import TestConfig, RebaselineServer
  37 from webkitpy.tool.servers import rebaselineserver
  38
  39
  40 class RebaselineTestTest(unittest.TestCase):
  41     def test_text_rebaseline_update(self):
  42         self._assertRebaseline(
  43             test_files=(
  44                 'fast/text-expected.txt',
  45                 'platform/mac/fast/text-expected.txt',
  46             ),
  47             results_files=(
  48                 'fast/text-actual.txt',
  49             ),
  50             test_name='fast/text.html',
  51             baseline_target='mac',
  52             baseline_move_to='none',
  53             expected_success=True,
  54             expected_log=[
  55                 'Rebaselining fast/text...',
  56                 '  Updating baselines for mac',
  57                 '    Updated text-expected.txt',
  58             ])
  59
  60     def test_text_rebaseline_new(self):
  61         self._assertRebaseline(
  62             test_files=(
  63                 'fast/text-expected.txt',
  64             ),
  65             results_files=(
  66                 'fast/text-actual.txt',
  67             ),
  68             test_name='fast/text.html',
  69             baseline_target='mac',
  70             baseline_move_to='none',
  71             expected_success=True,
  72             expected_log=[
  73                 'Rebaselining fast/text...',
  74                 '  Updating baselines for mac',
  75                 '    Updated text-expected.txt',
  76             ])
  77
  78     def test_text_rebaseline_move_no_op_1(self):
  79         self._assertRebaseline(
  80             test_files=(
  81                 'fast/text-expected.txt',
  82                 'platform/win/fast/text-expected.txt',
  83             ),
  84             results_files=(
  85                 'fast/text-actual.txt',
  86             ),
  87             test_name='fast/text.html',
  88             baseline_target='mac',
  89             baseline_move_to='mac-leopard',
  90             expected_success=True,
  91             expected_log=[
  92                 'Rebaselining fast/text...',
  93                 '  Updating baselines for mac',
  94                 '    Updated text-expected.txt',
  95             ])
  96
  97     def test_text_rebaseline_move_no_op_2(self):
  98         self._assertRebaseline(
  99             test_files=(
 100                 'fast/text-expected.txt',
 101                 'platform/mac/fast/text-expected.checksum',
 102             ),
 103             results_files=(
 104                 'fast/text-actual.txt',
 105             ),
 106             test_name='fast/text.html',
 107             baseline_target='mac',
 108             baseline_move_to='mac-leopard',
 109             expected_success=True,
 110             expected_log=[
 111                 'Rebaselining fast/text...',
 112                 '  Moving current mac baselines to mac-leopard',
 113                 '    No current baselines to move',
 114                 '  Updating baselines for mac',
 115                 '    Updated text-expected.txt',
 116             ])
 117
 118     def test_text_rebaseline_move(self):
 119         self._assertRebaseline(
 120             test_files=(
 121                 'fast/text-expected.txt',
 122                 'platform/mac/fast/text-expected.txt',
 123             ),
 124             results_files=(
 125                 'fast/text-actual.txt',
 126             ),
 127             test_name='fast/text.html',
 128             baseline_target='mac',
 129             baseline_move_to='mac-leopard',
 130             expected_success=True,
 131             expected_log=[
 132                 'Rebaselining fast/text...',
 133                 '  Moving current mac baselines to mac-leopard',
 134                 '    Moved text-expected.txt',
 135                 '  Updating baselines for mac',
 136                 '    Updated text-expected.txt',
 137             ])
 138
 139     def test_text_rebaseline_move_only_images(self):
 140         self._assertRebaseline(
 141             test_files=(
 142                 'fast/image-expected.txt',
 143                 'platform/mac/fast/image-expected.txt',
 144                 'platform/mac/fast/image-expected.png',
 145                 'platform/mac/fast/image-expected.checksum',
 146             ),
 147             results_files=(
 148                 'fast/image-actual.png',
 149                 'fast/image-actual.checksum',
 150             ),
 151             test_name='fast/image.html',
 152             baseline_target='mac',
 153             baseline_move_to='mac-leopard',
 154             expected_success=True,
 155             expected_log=[
 156                 'Rebaselining fast/image...',
 157                 '  Moving current mac baselines to mac-leopard',
 158                 '    Moved image-expected.checksum',
 159                 '    Moved image-expected.png',
 160                 '  Updating baselines for mac',
 161                 '    Updated image-expected.checksum',
 162                 '    Updated image-expected.png',
 163             ])
 164
 165     def test_text_rebaseline_move_already_exist(self):
 166         self._assertRebaseline(
 167             test_files=(
 168                 'fast/text-expected.txt',
 169                 'platform/mac-leopard/fast/text-expected.txt',
 170                 'platform/mac/fast/text-expected.txt',
 171             ),
 172             results_files=(
 173                 'fast/text-actual.txt',
 174             ),
 175             test_name='fast/text.html',
 176             baseline_target='mac',
 177             baseline_move_to='mac-leopard',
 178             expected_success=False,
 179             expected_log=[
 180                 'Rebaselining fast/text...',
 181                 '  Moving current mac baselines to mac-leopard',
 182                 '    Already had baselines in mac-leopard, could not move existing mac ones',
 183             ])
 184
 185     def test_image_rebaseline(self):
 186         self._assertRebaseline(
 187             test_files=(
 188                 'fast/image-expected.txt',
 189                 'platform/mac/fast/image-expected.png',
 190                 'platform/mac/fast/image-expected.checksum',
 191             ),
 192             results_files=(
 193                 'fast/image-actual.png',
 194                 'fast/image-actual.checksum',
 195             ),
 196             test_name='fast/image.html',
 197             baseline_target='mac',
 198             baseline_move_to='none',
 199             expected_success=True,
 200             expected_log=[
 201                 'Rebaselining fast/image...',
 202                 '  Updating baselines for mac',
 203                 '    Updated image-expected.checksum',
 204                 '    Updated image-expected.png',
 205             ])
 206
 207     def test_gather_baselines(self):
 208         example_json = layouttestresults_unittest.LayoutTestResultsTest.example_full_results_json
 209         results_json = json.loads(strip_json_wrapper(example_json))
 210         server = RebaselineServer()
 211         server._test_config = get_test_config()
 212         server._gather_baselines(results_json)
 213         self.assertEqual(results_json['tests']['svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html']['state'], 'needs_rebaseline')
 214         self.assertNotIn('prototype-chocolate.html', results_json['tests'])
 215
 216     def _assertRebaseline(self, test_files, results_files, test_name, baseline_target, baseline_move_to, expected_success, expected_log):
 217         log = []
 218         test_config = get_test_config(test_files, results_files)
 219         success = rebaselineserver._rebaseline_test(
 220             test_name,
 221             baseline_target,
 222             baseline_move_to,
 223             test_config,
 224             log=lambda l: log.append(l))
 225         self.assertEqual(expected_log, log)
 226         self.assertEqual(expected_success, success)
 227
 228
 229 class GetActualResultFilesTest(unittest.TestCase):
 230     def test(self):
 231         test_config = get_test_config(result_files=(
 232             'fast/text-actual.txt',
 233             'fast2/text-actual.txt',
 234             'fast/text2-actual.txt',
 235             'fast/text-notactual.txt',
 236         ))
 237         self.assertItemsEqual(
 238             ('text-actual.txt',),
 239             rebaselineserver._get_actual_result_files(
 240                 'fast/text.html', test_config))
 241
 242
 243 class GetBaselinesTest(unittest.TestCase):
 244     def test_no_baselines(self):
 245         self._assertBaselines(
 246             test_files=(),
 247             test_name='fast/missing.html',
 248             expected_baselines={})
 249
 250     def test_text_baselines(self):
 251         self._assertBaselines(
 252             test_files=(
 253                 'fast/text-expected.txt',
 254                 'platform/mac/fast/text-expected.txt',
 255             ),
 256             test_name='fast/text.html',
 257             expected_baselines={
 258                 'mac': {'.txt': True},
 259                 'base': {'.txt': False},
 260             })
 261
 262     def test_image_and_text_baselines(self):
 263         self._assertBaselines(
 264             test_files=(
 265                 'fast/image-expected.txt',
 266                 'platform/mac/fast/image-expected.png',
 267                 'platform/mac/fast/image-expected.checksum',
 268                 'platform/win/fast/image-expected.png',
 269                 'platform/win/fast/image-expected.checksum',
 270             ),
 271             test_name='fast/image.html',
 272             expected_baselines={
 273                 'base': {'.txt': True},
 274                 'mac': {'.checksum': True, '.png': True},
 275                 'win': {'.checksum': False, '.png': False},
 276             })
 277
 278     def test_extra_baselines(self):
 279         self._assertBaselines(
 280             test_files=(
 281                 'fast/text-expected.txt',
 282                 'platform/nosuchplatform/fast/text-expected.txt',
 283             ),
 284             test_name='fast/text.html',
 285             expected_baselines={'base': {'.txt': True}})
 286
 287     def _assertBaselines(self, test_files, test_name, expected_baselines):
 288         actual_baselines = rebaselineserver.get_test_baselines(test_name, get_test_config(test_files))
 289         self.assertEqual(expected_baselines, actual_baselines)
 290
 291
 292 def get_test_config(test_files=[], result_files=[]):
 293     host = MockHost()
 294     port = host.port_factory.get()
 295     layout_tests_directory = port.layout_tests_dir()
 296     results_directory = port.results_directory()
 297
 298     for file in test_files:
 299         host.filesystem.write_binary_file(host.filesystem.join(layout_tests_directory, file), '')
 300     for file in result_files:
 301         host.filesystem.write_binary_file(host.filesystem.join(results_directory, file), '')
 302
 303     class TestMacPort(Port):
 304         port_name = "mac"
 305         FALLBACK_PATHS = {'': ['mac']}
 306
 307     return TestConfig(
 308         TestMacPort(host, 'mac'),
 309         layout_tests_directory,
 310         results_directory,
 311         ('mac', 'mac-leopard', 'win', 'linux'),
 312         host)