1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 import webkitpy.thirdparty.unittest2 as unittest
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.system.systemhost_mock import MockSystemHost
34 from webkitpy.layout_tests import run_webkit_tests
35 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
36 from webkitpy.layout_tests.models import test_expectations
37 from webkitpy.layout_tests.models import test_failures
38 from webkitpy.layout_tests.models.test_run_results import TestRunResults
39 from webkitpy.layout_tests.models.test_input import TestInput
40 from webkitpy.layout_tests.models.test_results import TestResult
41 from webkitpy.layout_tests.port.test import TestPort
44 TestExpectations = test_expectations.TestExpectations
47 class FakePrinter(object):
51 def print_expected(self, run_results, get_tests_with_result_type):
54 def print_workers_and_shards(self, num_workers, num_shards, num_locked_shards):
57 def print_started_test(self, test_name):
60 def print_finished_test(self, result, expected, exp_str, got_str):
66 def write_update(self, msg):
73 class LockCheckingRunner(LayoutTestRunner):
74 def __init__(self, port, options, printer, tester, http_lock):
75 super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False)
76 self._finished_list_called = False
78 self._should_have_http_lock = http_lock
80 def handle_finished_list(self, source, list_name, num_tests, elapsed_time):
81 if not self._finished_list_called:
82 self._tester.assertEqual(list_name, 'locked_tests')
83 self._tester.assertTrue(self._remaining_locked_shards)
84 self._tester.assertTrue(self._has_http_lock is self._should_have_http_lock)
86 super(LockCheckingRunner, self).handle_finished_list(source, list_name, num_tests, elapsed_time)
88 if not self._finished_list_called:
89 self._tester.assertEqual(self._remaining_locked_shards, [])
90 self._tester.assertFalse(self._has_http_lock)
91 self._finished_list_called = True
94 class LayoutTestRunnerTests(unittest.TestCase):
95 def _runner(self, port=None):
96 # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
97 options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
98 options.child_processes = '1'
101 port = port or host.port_factory.get(options.platform, options=options)
102 return LockCheckingRunner(port, options, FakePrinter(), self, True)
104 def _run_tests(self, runner, tests):
105 test_inputs = [TestInput(test, 6000) for test in tests]
106 expectations = TestExpectations(runner._port, tests)
107 runner.run_tests(expectations, test_inputs, set(), num_workers=1, retrying=False)
109 def test_interrupt_if_at_failure_limits(self):
110 runner = self._runner()
111 runner._options.exit_after_n_failures = None
112 runner._options.exit_after_n_crashes_or_times = None
113 test_names = ['passes/text.html', 'passes/image.html']
114 runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names]
116 run_results = TestRunResults(TestExpectations(runner._port, test_names), len(test_names))
117 run_results.unexpected_failures = 100
118 run_results.unexpected_crashes = 50
119 run_results.unexpected_timeouts = 50
120 # No exception when the exit_after* options are None.
121 runner._interrupt_if_at_failure_limits(run_results)
123 # No exception when we haven't hit the limit yet.
124 runner._options.exit_after_n_failures = 101
125 runner._options.exit_after_n_crashes_or_timeouts = 101
126 runner._interrupt_if_at_failure_limits(run_results)
128 # Interrupt if we've exceeded either limit:
129 runner._options.exit_after_n_crashes_or_timeouts = 10
130 self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
131 self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP)
132 self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP)
134 runner._options.exit_after_n_crashes_or_timeouts = None
135 runner._options.exit_after_n_failures = 10
136 exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
138 def test_update_summary_with_result(self):
139 # Reftests expected to be image mismatch should be respected when pixel_tests=False.
140 runner = self._runner()
141 runner._options.pixel_tests = False
142 test = 'failures/expected/reftest.html'
143 expectations = TestExpectations(runner._port, tests=[test])
144 runner._expectations = expectations
146 run_results = TestRunResults(expectations, 1)
147 result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!='])
148 runner._update_summary_with_result(run_results, result)
149 self.assertEqual(1, run_results.expected)
150 self.assertEqual(0, run_results.unexpected)
152 run_results = TestRunResults(expectations, 1)
153 result = TestResult(test_name=test, failures=[], reftest_type=['=='])
154 runner._update_summary_with_result(run_results, result)
155 self.assertEqual(0, run_results.expected)
156 self.assertEqual(1, run_results.unexpected)
159 class SharderTests(unittest.TestCase):
162 "http/tests/websocket/tests/unicode.htm",
163 "animations/keyframes.html",
164 "http/tests/security/view-source-no-refresh.html",
165 "http/tests/websocket/tests/websocket-protocol-ignored.html",
166 "fast/css/display-none-inline-style-change-crash.html",
167 "http/tests/xmlhttprequest/supported-xml-content-types.html",
168 "dom/html/level2/html/HTMLAnchorElement03.html",
169 "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
170 "dom/html/level2/html/HTMLAnchorElement06.html",
171 "perf/object-keys.html",
172 "virtual/threaded/dir/test.html",
173 "virtual/threaded/fast/foo/test.html",
176 def get_test_input(self, test_file):
177 return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf')))
179 def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=1):
180 port = TestPort(MockSystemHost())
181 self.sharder = Sharder(port.split_test, max_locked_shards)
182 test_list = test_list or self.test_list
183 return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)
185 def assert_shards(self, actual_shards, expected_shard_names):
186 self.assertEqual(len(actual_shards), len(expected_shard_names))
187 for i, shard in enumerate(actual_shards):
188 expected_shard_name, expected_test_names = expected_shard_names[i]
189 self.assertEqual(shard.name, expected_shard_name)
190 self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
193 def test_shard_by_dir(self):
194 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False)
196 # Note that although there are tests in multiple dirs that need locks,
197 # they are crammed into a single shard in order to reduce the # of
198 # workers hitting the server at once.
199 self.assert_shards(locked,
201 ['http/tests/security/view-source-no-refresh.html',
202 'http/tests/websocket/tests/unicode.htm',
203 'http/tests/websocket/tests/websocket-protocol-ignored.html',
204 'http/tests/xmlhttprequest/supported-xml-content-types.html',
205 'perf/object-keys.html'])])
206 self.assert_shards(unlocked,
207 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
208 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
209 ('animations', ['animations/keyframes.html']),
210 ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
211 'dom/html/level2/html/HTMLAnchorElement06.html']),
212 ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
213 ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
215 def test_shard_every_file(self):
216 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2)
217 self.assert_shards(locked,
219 ['http/tests/websocket/tests/unicode.htm',
220 'http/tests/security/view-source-no-refresh.html',
221 'http/tests/websocket/tests/websocket-protocol-ignored.html']),
223 ['http/tests/xmlhttprequest/supported-xml-content-types.html',
224 'perf/object-keys.html'])]),
225 self.assert_shards(unlocked,
226 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']),
227 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']),
228 ('.', ['animations/keyframes.html']),
229 ('.', ['fast/css/display-none-inline-style-change-crash.html']),
230 ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
231 ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
232 ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
234 def test_shard_in_two(self):
235 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False)
236 self.assert_shards(locked,
238 ['http/tests/websocket/tests/unicode.htm',
239 'http/tests/security/view-source-no-refresh.html',
240 'http/tests/websocket/tests/websocket-protocol-ignored.html',
241 'http/tests/xmlhttprequest/supported-xml-content-types.html',
242 'perf/object-keys.html'])])
243 self.assert_shards(unlocked,
245 ['animations/keyframes.html',
246 'fast/css/display-none-inline-style-change-crash.html',
247 'dom/html/level2/html/HTMLAnchorElement03.html',
248 'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
249 'dom/html/level2/html/HTMLAnchorElement06.html',
250 'virtual/threaded/dir/test.html',
251 'virtual/threaded/fast/foo/test.html'])])
253 def test_shard_in_two_has_no_locked_shards(self):
254 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
255 test_list=['animations/keyframe.html'])
256 self.assertEqual(len(locked), 0)
257 self.assertEqual(len(unlocked), 1)
259 def test_shard_in_two_has_no_unlocked_shards(self):
260 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
261 test_list=['http/tests/websocket/tests/unicode.htm'])
262 self.assertEqual(len(locked), 1)
263 self.assertEqual(len(unlocked), 0)
265 def test_multiple_locked_shards(self):
266 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2)
267 self.assert_shards(locked,
269 ['http/tests/security/view-source-no-refresh.html',
270 'http/tests/websocket/tests/unicode.htm',
271 'http/tests/websocket/tests/websocket-protocol-ignored.html']),
273 ['http/tests/xmlhttprequest/supported-xml-content-types.html',
274 'perf/object-keys.html'])])
276 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False)
277 self.assert_shards(locked,
279 ['http/tests/security/view-source-no-refresh.html',
280 'http/tests/websocket/tests/unicode.htm',
281 'http/tests/websocket/tests/websocket-protocol-ignored.html',
282 'http/tests/xmlhttprequest/supported-xml-content-types.html',
283 'perf/object-keys.html'])])