1 # Copyright (C) 2009 Google Inc. All rights reserved.
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
13 # * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 from webkitpy.common.net.layouttestresults import LayoutTestResults
32 from webkitpy.common.net.buildbot import BuildBot, Builder, Build
33 from webkitpy.layout_tests.models import test_results
34 from webkitpy.layout_tests.models import test_failures
35 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
38 class BuilderTest(unittest.TestCase):
39 def _mock_test_result(self, testname):
40 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()])
42 def _install_fetch_build(self, failure):
43 def _mock_fetch_build(build_number):
46 build_number=build_number,
47 revision=build_number + 1000,
48 is_green=build_number < 4
50 results = [self._mock_test_result(testname) for testname in failure(build_number)]
51 build._layout_test_results = LayoutTestResults(results)
53 self.builder._fetch_build = _mock_fetch_build
56 self.buildbot = BuildBot()
57 self.builder = Builder(u"Test Builder \u2661", self.buildbot)
58 self._install_fetch_build(lambda build_number: ["test1", "test2"])
60 def test_find_regression_window(self):
61 regression_window = self.builder.find_regression_window(self.builder.build(10))
62 self.assertEqual(regression_window.build_before_failure().revision(), 1003)
63 self.assertEqual(regression_window.failing_build().revision(), 1004)
65 regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
66 self.assertEqual(regression_window.build_before_failure(), None)
67 self.assertEqual(regression_window.failing_build().revision(), 1008)
69 def test_none_build(self):
70 self.builder._fetch_build = lambda build_number: None
71 regression_window = self.builder.find_regression_window(self.builder.build(10))
72 self.assertEqual(regression_window.build_before_failure(), None)
73 self.assertEqual(regression_window.failing_build(), None)
75 def test_flaky_tests(self):
76 self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
77 regression_window = self.builder.find_regression_window(self.builder.build(10))
78 self.assertEqual(regression_window.build_before_failure().revision(), 1009)
79 self.assertEqual(regression_window.failing_build().revision(), 1010)
81 def test_failure_and_flaky(self):
82 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
83 regression_window = self.builder.find_regression_window(self.builder.build(10))
84 self.assertEqual(regression_window.build_before_failure().revision(), 1003)
85 self.assertEqual(regression_window.failing_build().revision(), 1004)
87 def test_no_results(self):
88 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
89 regression_window = self.builder.find_regression_window(self.builder.build(10))
90 self.assertEqual(regression_window.build_before_failure().revision(), 1003)
91 self.assertEqual(regression_window.failing_build().revision(), 1004)
93 def test_failure_after_flaky(self):
94 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
95 regression_window = self.builder.find_regression_window(self.builder.build(10))
96 self.assertEqual(regression_window.build_before_failure().revision(), 1006)
97 self.assertEqual(regression_window.failing_build().revision(), 1007)
99 def test_find_blameworthy_regression_window(self):
100 self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
101 self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
102 # Flakey test avoidance requires at least 2 red builds:
103 self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
104 self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
106 self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
108 def test_build_caching(self):
109 self.assertEqual(self.builder.build(10), self.builder.build(10))
111 def test_build_and_revision_for_filename(self):
113 "r47483 (1)/" : (47483, 1),
114 "r47483 (1).zip" : (47483, 1),
116 for filename, revision_and_build in expectations.items():
117 self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
119 def test_fetch_build(self):
120 buildbot = BuildBot()
121 builder = Builder(u"Test Builder \u2661", buildbot)
123 def mock_fetch_build_dictionary(self, build_number):
126 "revision": None, # revision=None means a trunk build started from the force-build button on the builder page.
128 "number": int(build_number),
129 # Intentionally missing the 'results' key, meaning it's a "pass" build.
131 return build_dictionary
132 buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
133 self.assertNotEqual(builder._fetch_build(1), None)
136 class BuildTest(unittest.TestCase):
137 def test_layout_test_results(self):
138 buildbot = BuildBot()
139 builder = Builder(u"Foo Builder (test)", buildbot)
140 build = Build(builder, None, None, None)
141 build._fetch_file_from_results = lambda file_name: None
142 # Test that layout_test_results() returns None if the fetch fails.
143 self.assertEqual(build.layout_test_results(), None)
146 class BuildBotTest(unittest.TestCase):
148 _example_one_box_status = '''
151 <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td>
152 <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td>
153 <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td>
155 <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td>
156 <td class="LastBuild box" >no build</td>
157 <td align="center" class="Activity building">building<br />< 1 min</td>
159 <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td>
160 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td>
161 <td align="center" class="Activity idle">idle<br />3 pending</td>
163 <td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td>
164 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td>
165 <td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td>
168 _expected_example_one_box_parsings = [
171 'build_number' : 3693,
172 'name': u'Windows Debug (Tests)',
173 'built_revision': 47380,
174 'activity': 'building',
179 'build_number' : None,
180 'name': u'SnowLeopard Intel Release',
181 'built_revision': None,
182 'activity': 'building',
187 'build_number' : 654,
188 'name': u'Qt Linux Release',
189 'built_revision': 47383,
195 'build_number' : 2090,
196 'name': u'Qt Windows 32-bit Debug',
197 'built_revision': 60563,
198 'activity': 'building',
203 def test_status_parsing(self):
204 buildbot = BuildBot()
206 soup = BeautifulSoup(self._example_one_box_status)
207 status_table = soup.find("table")
208 input_rows = status_table.findAll('tr')
210 for x in range(len(input_rows)):
211 status_row = input_rows[x]
212 expected_parsing = self._expected_example_one_box_parsings[x]
214 builder = buildbot._parse_builder_status_from_row(status_row)
216 # Make sure we aren't parsing more or less than we expect
217 self.assertEquals(builder.keys(), expected_parsing.keys())
219 for key, expected_value in expected_parsing.items():
220 self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value)))
222 def test_core_builder_methods(self):
223 buildbot = BuildBot()
225 # Override builder_statuses function to not touch the network.
226 def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to.
227 return BuildBotTest._expected_example_one_box_parsings
228 buildbot.builder_statuses = example_builder_statuses
230 buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ]
231 self.assertEquals(buildbot.red_core_builders_names(), [])
232 self.assertTrue(buildbot.core_builders_are_green())
234 buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ]
235 self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ])
236 self.assertFalse(buildbot.core_builders_are_green())
238 def test_builder_name_regexps(self):
239 buildbot = BuildBot()
241 # For complete testing, this list should match the list of builders at build.webkit.org:
243 {'name': u'Leopard Intel Release (Build)', },
244 {'name': u'Leopard Intel Release (Tests)', },
245 {'name': u'Leopard Intel Debug (Build)', },
246 {'name': u'Leopard Intel Debug (Tests)', },
247 {'name': u'SnowLeopard Intel Release (Build)', },
248 {'name': u'SnowLeopard Intel Release (Tests)', },
249 {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', },
250 {'name': u'SnowLeopard Intel Leaks', },
251 {'name': u'Windows Release (Build)', },
252 {'name': u'Windows 7 Release (Tests)', },
253 {'name': u'Windows Debug (Build)', },
254 {'name': u'Windows XP Debug (Tests)', },
255 {'name': u'Windows 7 Release (WebKit2 Tests)', },
256 {'name': u'GTK Linux 32-bit Release', },
257 {'name': u'GTK Linux 32-bit Debug', },
258 {'name': u'GTK Linux 64-bit Debug', },
259 {'name': u'Qt Linux Release', },
260 {'name': u'Qt Linux Release minimal', },
261 {'name': u'Qt Linux ARMv7 Release', },
262 {'name': u'Qt Windows 32-bit Release', },
263 {'name': u'Qt Windows 32-bit Debug', },
264 {'name': u'Chromium Android Release', },
265 {'name': u'Chromium Win Release', },
266 {'name': u'Chromium Win Release (Tests)', },
267 {'name': u'Chromium Mac Release', },
268 {'name': u'Chromium Mac Release (Tests)', },
269 {'name': u'Chromium Linux Release', },
270 {'name': u'Chromium Linux Release (Tests)', },
271 {'name': u'Leopard Intel Release (NRWT)', },
272 {'name': u'SnowLeopard Intel Release (NRWT)', },
273 {'name': u'New run-webkit-tests', },
274 {'name': u'WinCairo Debug (Build)', },
275 {'name': u'WinCE Release (Build)', },
276 {'name': u'EFL Linux Release (Build)', },
279 "SnowLeopard.*Build",
280 "SnowLeopard.*\(Test",
281 "SnowLeopard.*\(WebKit2 Test",
282 "Leopard.*\((?:Build|Test)",
288 "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken.
290 "Chromium.*(Mac|Linux|Win).*Release$",
291 "Chromium.*(Mac|Linux|Win).*Release.*\(Tests",
293 expected_builders = [
294 {'name': u'Leopard Intel Release (Build)', },
295 {'name': u'Leopard Intel Release (Tests)', },
296 {'name': u'Leopard Intel Debug (Build)', },
297 {'name': u'Leopard Intel Debug (Tests)', },
298 {'name': u'SnowLeopard Intel Release (Build)', },
299 {'name': u'SnowLeopard Intel Release (Tests)', },
300 {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', },
301 {'name': u'Windows Release (Build)', },
302 {'name': u'Windows 7 Release (Tests)', },
303 {'name': u'Windows Debug (Build)', },
304 {'name': u'Windows XP Debug (Tests)', },
305 {'name': u'GTK Linux 32-bit Release', },
306 {'name': u'GTK Linux 32-bit Debug', },
307 {'name': u'GTK Linux 64-bit Debug', },
308 {'name': u'Qt Linux Release', },
309 {'name': u'Qt Linux Release minimal', },
310 {'name': u'Qt Linux ARMv7 Release', },
311 {'name': u'Qt Windows 32-bit Release', },
312 {'name': u'Qt Windows 32-bit Debug', },
313 {'name': u'Chromium Win Release', },
314 {'name': u'Chromium Win Release (Tests)', },
315 {'name': u'Chromium Mac Release', },
316 {'name': u'Chromium Mac Release (Tests)', },
317 {'name': u'Chromium Linux Release', },
318 {'name': u'Chromium Linux Release (Tests)', },
319 {'name': u'WinCE Release (Build)', },
320 {'name': u'EFL Linux Release (Build)', },
323 # This test should probably be updated if the default regexp list changes
324 self.assertEquals(buildbot.core_builder_names_regexps, name_regexps)
326 builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps)
327 self.assertEquals(builders, expected_builders)
329 def test_builder_with_name(self):
330 buildbot = BuildBot()
332 builder = buildbot.builder_with_name("Test Builder")
333 self.assertEqual(builder.name(), "Test Builder")
334 self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder")
335 self.assertEqual(builder.url_encoded_name(), "Test%20Builder")
336 self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder")
338 # Override _fetch_build_dictionary function to not touch the network.
339 def mock_fetch_build_dictionary(self, build_number):
342 "revision" : 2 * build_number,
344 "number" : int(build_number),
345 "results" : build_number % 2, # 0 means pass
347 return build_dictionary
348 buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
350 build = builder.build(10)
351 self.assertEqual(build.builder(), builder)
352 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10")
353 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29")
354 self.assertEqual(build.revision(), 20)
355 self.assertEqual(build.is_green(), True)
357 build = build.previous_build()
358 self.assertEqual(build.builder(), builder)
359 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9")
360 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29")
361 self.assertEqual(build.revision(), 18)
362 self.assertEqual(build.is_green(), False)
364 self.assertEqual(builder.build(None), None)
366 _example_directory_listing = '''
367 <h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1>
373 <th>Content type</th>
374 <th>Content encoding</th>
376 <tr class="directory ">
377 <td><a href="r47483%20%281%29/"><b>r47483 (1)/</b></a></td>
379 <td><b>[Directory]</b></td>
382 <tr class="file alt">
383 <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td>
385 <td>[application/zip]</td>
391 "filename" : "r47483 (1)/",
393 "type" : "[Directory]",
397 "filename" : "r47484 (2).zip",
399 "type" : "[application/zip]",
404 def test_parse_build_to_revision_map(self):
405 buildbot = BuildBot()
406 files = buildbot._parse_twisted_directory_listing(self._example_directory_listing)
407 self.assertEqual(self._expected_files, files)
410 # Ordered from newest (highest number) to oldest.
423 def _build_from_fake(self, fake_builder, index):
424 if index >= len(fake_builder):
426 fake_build = fake_builder[index]
428 builder=fake_builder,
430 revision=fake_build[0],
431 is_green=fake_build[1],
433 def mock_previous_build():
434 return self._build_from_fake(fake_builder, index + 1)
435 build.previous_build = mock_previous_build
438 def _fake_builds_at_index(self, index):
439 return [self._build_from_fake(builder, index) for builder in self.fake_builders]
441 def test_last_green_revision(self):
442 buildbot = BuildBot()
443 def mock_builds_from_builders(only_core_builders):
444 return self._fake_builds_at_index(0)
445 buildbot._latest_builds_from_builders = mock_builds_from_builders
446 self.assertEqual(buildbot.last_green_revision(), 1)
448 def _fetch_build(self, build_number):
449 if build_number == 5:
450 return "correct build"
453 def _fetch_revision_to_build_map(self):
454 return {'r5': 5, 'r2': 2, 'r3': 3}
456 def test_latest_cached_build(self):
457 b = Builder('builder', BuildBot())
458 b._fetch_build = self._fetch_build
459 b._fetch_revision_to_build_map = self._fetch_revision_to_build_map
460 self.assertEquals("correct build", b.latest_cached_build())
462 def results_url(self):
465 def test_results_zip_url(self):
466 b = Build(None, 123, 123, False)
467 b.results_url = self.results_url
468 self.assertEquals("some-url.zip", b.results_zip_url())
471 if __name__ == '__main__':