src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py

   1 # Copyright (C) 2012 Google Inc. All rights reserved.
   2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 import unittest
  31
  32 from webkitpy.common.host_mock import MockHost
  33 from webkitpy.common.system.systemhost_mock import MockSystemHost
  34 from webkitpy.layout_tests import run_webkit_tests
  35 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
  36 from webkitpy.layout_tests.models import test_expectations
  37 from webkitpy.layout_tests.models import test_failures
  38 from webkitpy.layout_tests.models.test_run_results import TestRunResults
  39 from webkitpy.layout_tests.models.test_input import TestInput
  40 from webkitpy.layout_tests.models.test_results import TestResult
  41 from webkitpy.layout_tests.port.test import TestPort
  42
  43
  44 TestExpectations = test_expectations.TestExpectations
  45
  46
  47 class FakePrinter(object):
  48     num_completed = 0
  49     num_tests = 0
  50
  51     def print_expected(self, run_results, get_tests_with_result_type):
  52         pass
  53
  54     def print_workers_and_shards(self, num_workers, num_shards, num_locked_shards):
  55         pass
  56
  57     def print_started_test(self, test_name):
  58         pass
  59
  60     def print_finished_test(self, result, expected, exp_str, got_str):
  61         pass
  62
  63     def write(self, msg):
  64         pass
  65
  66     def write_update(self, msg):
  67         pass
  68
  69     def flush(self):
  70         pass
  71
  72
  73 class LockCheckingRunner(LayoutTestRunner):
  74     def __init__(self, port, options, printer, tester, http_lock):
  75         super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False)
  76         self._finished_list_called = False
  77         self._tester = tester
  78         self._should_have_http_lock = http_lock
  79
  80     def handle_finished_list(self, source, list_name, num_tests, elapsed_time):
  81         if not self._finished_list_called:
  82             self._tester.assertEqual(list_name, 'locked_tests')
  83             self._tester.assertTrue(self._remaining_locked_shards)
  84             self._tester.assertTrue(self._has_http_lock is self._should_have_http_lock)
  85
  86         super(LockCheckingRunner, self).handle_finished_list(source, list_name, num_tests, elapsed_time)
  87
  88         if not self._finished_list_called:
  89             self._tester.assertEqual(self._remaining_locked_shards, [])
  90             self._tester.assertFalse(self._has_http_lock)
  91             self._finished_list_called = True
  92
  93
  94 class LayoutTestRunnerTests(unittest.TestCase):
  95     def _runner(self, port=None):
  96         # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
  97         options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
  98         options.child_processes = '1'
  99
 100         host = MockHost()
 101         port = port or host.port_factory.get(options.platform, options=options)
 102         return LockCheckingRunner(port, options, FakePrinter(), self, True)
 103
 104     def _run_tests(self, runner, tests):
 105         test_inputs = [TestInput(test, 6000) for test in tests]
 106         expectations = TestExpectations(runner._port, tests)
 107         runner.run_tests(expectations, test_inputs, set(), num_workers=1, retrying=False)
 108
 109     def test_interrupt_if_at_failure_limits(self):
 110         runner = self._runner()
 111         runner._options.exit_after_n_failures = None
 112         runner._options.exit_after_n_crashes_or_times = None
 113         test_names = ['passes/text.html', 'passes/image.html']
 114         runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names]
 115
 116         run_results = TestRunResults(TestExpectations(runner._port, test_names), len(test_names))
 117         run_results.unexpected_failures = 100
 118         run_results.unexpected_crashes = 50
 119         run_results.unexpected_timeouts = 50
 120         # No exception when the exit_after* options are None.
 121         runner._interrupt_if_at_failure_limits(run_results)
 122
 123         # No exception when we haven't hit the limit yet.
 124         runner._options.exit_after_n_failures = 101
 125         runner._options.exit_after_n_crashes_or_timeouts = 101
 126         runner._interrupt_if_at_failure_limits(run_results)
 127
 128         # Interrupt if we've exceeded either limit:
 129         runner._options.exit_after_n_crashes_or_timeouts = 10
 130         self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
 131         self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP)
 132         self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP)
 133
 134         runner._options.exit_after_n_crashes_or_timeouts = None
 135         runner._options.exit_after_n_failures = 10
 136         exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
 137
 138     def test_update_summary_with_result(self):
 139         # Reftests expected to be image mismatch should be respected when pixel_tests=False.
 140         runner = self._runner()
 141         runner._options.pixel_tests = False
 142         test = 'failures/expected/reftest.html'
 143         expectations = TestExpectations(runner._port, tests=[test])
 144         runner._expectations = expectations
 145
 146         run_results = TestRunResults(expectations, 1)
 147         result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!='])
 148         runner._update_summary_with_result(run_results, result)
 149         self.assertEqual(1, run_results.expected)
 150         self.assertEqual(0, run_results.unexpected)
 151
 152         run_results = TestRunResults(expectations, 1)
 153         result = TestResult(test_name=test, failures=[], reftest_type=['=='])
 154         runner._update_summary_with_result(run_results, result)
 155         self.assertEqual(0, run_results.expected)
 156         self.assertEqual(1, run_results.unexpected)
 157
 158
 159 class SharderTests(unittest.TestCase):
 160
 161     test_list = [
 162         "http/tests/websocket/tests/unicode.htm",
 163         "animations/keyframes.html",
 164         "http/tests/security/view-source-no-refresh.html",
 165         "http/tests/websocket/tests/websocket-protocol-ignored.html",
 166         "fast/css/display-none-inline-style-change-crash.html",
 167         "http/tests/xmlhttprequest/supported-xml-content-types.html",
 168         "dom/html/level2/html/HTMLAnchorElement03.html",
 169         "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
 170         "dom/html/level2/html/HTMLAnchorElement06.html",
 171         "perf/object-keys.html",
 172         "virtual/threaded/dir/test.html",
 173         "virtual/threaded/fast/foo/test.html",
 174     ]
 175
 176     def get_test_input(self, test_file):
 177         return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')))
 178
 179     def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1):
 180         port = TestPort(MockSystemHost())
 181         self.sharder = Sharder(port.split_test, max_locked_shards)
 182         test_list = test_list or self.test_list
 183         return self.sharder.shard_tests([self.get_test_input(test) for test in test_list],
 184             num_workers, fully_parallel, run_singly)
 185
 186     def assert_shards(self, actual_shards, expected_shard_names):
 187         self.assertEqual(len(actual_shards), len(expected_shard_names))
 188         for i, shard in enumerate(actual_shards):
 189             expected_shard_name, expected_test_names = expected_shard_names[i]
 190             self.assertEqual(shard.name, expected_shard_name)
 191             self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
 192                               expected_test_names)
 193
 194     def test_shard_by_dir(self):
 195         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False)
 196
 197         # Note that although there are tests in multiple dirs that need locks,
 198         # they are crammed into a single shard in order to reduce the # of
 199         # workers hitting the server at once.
 200         self.assert_shards(locked,
 201              [('locked_shard_1',
 202                ['http/tests/security/view-source-no-refresh.html',
 203                 'http/tests/websocket/tests/unicode.htm',
 204                 'http/tests/websocket/tests/websocket-protocol-ignored.html',
 205                 'http/tests/xmlhttprequest/supported-xml-content-types.html',
 206                 'perf/object-keys.html'])])
 207         self.assert_shards(unlocked,
 208             [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
 209              ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
 210              ('animations', ['animations/keyframes.html']),
 211              ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
 212                                       'dom/html/level2/html/HTMLAnchorElement06.html']),
 213              ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
 214              ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
 215
 216     def test_shard_every_file(self):
 217         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False)
 218         self.assert_shards(locked,
 219             [('locked_shard_1',
 220               ['http/tests/websocket/tests/unicode.htm',
 221                'http/tests/security/view-source-no-refresh.html',
 222                'http/tests/websocket/tests/websocket-protocol-ignored.html']),
 223              ('locked_shard_2',
 224               ['http/tests/xmlhttprequest/supported-xml-content-types.html',
 225                'perf/object-keys.html'])]),
 226         self.assert_shards(unlocked,
 227             [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
 228              ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
 229              ('.', ['animations/keyframes.html']),
 230              ('.', ['fast/css/display-none-inline-style-change-crash.html']),
 231              ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
 232              ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
 233              ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
 234
 235     def test_shard_in_two(self):
 236         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False)
 237         self.assert_shards(locked,
 238             [('locked_tests',
 239               ['http/tests/websocket/tests/unicode.htm',
 240                'http/tests/security/view-source-no-refresh.html',
 241                'http/tests/websocket/tests/websocket-protocol-ignored.html',
 242                'http/tests/xmlhttprequest/supported-xml-content-types.html',
 243                'perf/object-keys.html'])])
 244         self.assert_shards(unlocked,
 245             [('unlocked_tests',
 246               ['animations/keyframes.html',
 247                'fast/css/display-none-inline-style-change-crash.html',
 248                'dom/html/level2/html/HTMLAnchorElement03.html',
 249                'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
 250                'dom/html/level2/html/HTMLAnchorElement06.html',
 251                'virtual/threaded/dir/test.html',
 252                'virtual/threaded/fast/foo/test.html'])])
 253
 254     def test_shard_in_two_has_no_locked_shards(self):
 255         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
 256              test_list=['animations/keyframe.html'])
 257         self.assertEqual(len(locked), 0)
 258         self.assertEqual(len(unlocked), 1)
 259
 260     def test_shard_in_two_has_no_unlocked_shards(self):
 261         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False,
 262              test_list=['http/tests/websocket/tests/unicode.htm'])
 263         self.assertEqual(len(locked), 1)
 264         self.assertEqual(len(unlocked), 0)
 265
 266     def test_multiple_locked_shards(self):
 267         locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False)
 268         self.assert_shards(locked,
 269             [('locked_shard_1',
 270               ['http/tests/security/view-source-no-refresh.html',
 271                'http/tests/websocket/tests/unicode.htm',
 272                'http/tests/websocket/tests/websocket-protocol-ignored.html']),
 273              ('locked_shard_2',
 274               ['http/tests/xmlhttprequest/supported-xml-content-types.html',
 275                'perf/object-keys.html'])])
 276
 277         locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False)
 278         self.assert_shards(locked,
 279             [('locked_shard_1',
 280               ['http/tests/security/view-source-no-refresh.html',
 281                'http/tests/websocket/tests/unicode.htm',
 282                'http/tests/websocket/tests/websocket-protocol-ignored.html',
 283                'http/tests/xmlhttprequest/supported-xml-content-types.html',
 284                'perf/object-keys.html'])])
 285
 286     def test_virtual_shards(self):
 287         # With run_singly=False, we try to keep all of the tests in a virtual suite together even
 288         # when fully_parallel=True, so that we don't restart every time the command line args change.
 289         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False,
 290                 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
 291         self.assert_shards(unlocked,
 292             [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])])
 293
 294         # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism.
 295         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True,
 296                 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])
 297         self.assert_shards(unlocked,
 298             [('.', ['virtual/foo/bar1.html']),
 299              ('.', ['virtual/foo/bar2.html'])])