Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py

   1 # Copyright (C) 2011 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 try:
  30     import json
  31 except ImportError:
  32     # python 2.5 compatibility
  33     import webkitpy.thirdparty.simplejson as json
  34
  35 import unittest
  36
  37 from webkitpy.common.system.outputcapture import OutputCapture
  38 from webkitpy.layout_tests.models.test_configuration import *
  39 from webkitpy.thirdparty.mock import Mock
  40 from webkitpy.tool.mocktool import MockTool
  41 from webkitpy.common.system.executive_mock import MockExecutive
  42 from webkitpy.common.host_mock import MockHost
  43 from webkitpy.tool.servers.gardeningserver import *
  44
  45
  46 class TestPortFactory(object):
  47     # FIXME: Why is this a class method?
  48     @classmethod
  49     def create(cls):
  50         host = MockHost()
  51         return host.port_factory.get("test-win-xp")
  52
  53     @classmethod
  54     def path_to_test_expectations_file(cls):
  55         return cls.create().path_to_test_expectations_file()
  56
  57
  58 class MockServer(object):
  59     def __init__(self):
  60         self.tool = MockTool()
  61         self.tool.executive = MockExecutive(should_log=True)
  62         self.tool.filesystem.files[TestPortFactory.path_to_test_expectations_file()] = ""
  63
  64
  65 # The real GardeningHTTPRequestHandler has a constructor that's too hard to
  66 # call in a unit test, so we create a subclass that's easier to constrcut.
  67 class TestGardeningHTTPRequestHandler(GardeningHTTPRequestHandler):
  68     def __init__(self, server):
  69         self.server = server
  70         self.body = None
  71
  72     def _expectations_updater(self):
  73         return GardeningExpectationsUpdater(self.server.tool, TestPortFactory.create())
  74
  75     def _read_entity_body(self):
  76         return self.body if self.body else ''
  77
  78     def _serve_text(self, text):
  79         print "== Begin Response =="
  80         print text
  81         print "== End Response =="
  82
  83     def _serve_json(self, json_object):
  84         print "== Begin JSON Response =="
  85         print json.dumps(json_object)
  86         print "== End JSON Response =="
  87
  88
  89 class BuildCoverageExtrapolatorTest(unittest.TestCase):
  90     def test_extrapolate(self):
  91         # FIXME: Make this test not rely on actual (not mock) port objects.
  92         host = MockHost()
  93         port = host.port_factory.get('chromium-win-win7', None)
  94         converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
  95         extrapolator = BuildCoverageExtrapolator(converter)
  96         self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win"), set([TestConfiguration(version='xp', architecture='x86', build_type='release', graphics_type='cpu')]))
  97         self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Vista"), set([
  98             TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='cpu'),
  99             TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='gpu'),
 100             TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='gpu'),
 101             TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='cpu')]))
 102         self.assertRaises(KeyError, extrapolator.extrapolate_test_configurations, "Potato")
 103
 104
 105 class GardeningExpectationsUpdaterTest(unittest.TestCase):
 106     def __init__(self, testFunc):
 107         self.tool = MockTool()
 108         self.tool.executive = MockExecutive(should_log=True)
 109         self.tool.filesystem.files[TestPortFactory.path_to_test_expectations_file()] = ""
 110         unittest.TestCase.__init__(self, testFunc)
 111
 112     def assert_update(self, failure_info_list, expectations_before=None, expectations_after=None, expected_exception=None):
 113         updater = GardeningExpectationsUpdater(self.tool, TestPortFactory.create())
 114         path_to_test_expectations_file = TestPortFactory.path_to_test_expectations_file()
 115         self.tool.filesystem.files[path_to_test_expectations_file] = expectations_before or ""
 116         if expected_exception:
 117             self.assertRaises(expected_exception, updater.update_expectations, (failure_info_list))
 118         else:
 119             updater.update_expectations(failure_info_list)
 120             self.assertEquals(self.tool.filesystem.files[path_to_test_expectations_file], expectations_after)
 121
 122     def test_empty_expectations(self):
 123         failure_info_list = []
 124         expectations_before = ""
 125         expectations_after = ""
 126         self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
 127
 128     def test_unknown_builder(self):
 129         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Bob", "failureTypeList": ["IMAGE"]}]
 130         self.assert_update(failure_info_list, expected_exception=KeyError)
 131
 132     def test_empty_failure_type_list(self):
 133         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": []}]
 134         self.assert_update(failure_info_list, expected_exception=AssertionError)
 135
 136     def test_empty_test_name(self):
 137         failure_info_list = [{"testName": "", "builderName": "Webkit Win", "failureTypeList": ["TEXT"]}]
 138         self.assert_update(failure_info_list, expected_exception=AssertionError)
 139
 140     def test_unknown_failure_type(self):
 141         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE", "EXPLODE"]}]
 142         expectations_before = ""
 143         expectations_after = "\nBUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
 144         self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
 145
 146     def test_add_new_expectation(self):
 147         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE"]}]
 148         expectations_before = ""
 149         expectations_after = "\nBUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
 150         self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
 151
 152     def test_replace_old_expectation(self):
 153         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE"]}]
 154         expectations_before = "BUG_OLD XP RELEASE CPU : failures/expected/image.html = TEXT"
 155         expectations_after = "BUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
 156         self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
 157
 158     def test_pass_expectation(self):
 159         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["PASS"]}]
 160         expectations_before = "BUG_OLD XP RELEASE CPU : failures/expected/image.html = TEXT"
 161         expectations_after = ""
 162         self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
 163
 164     def test_supplement_old_expectation(self):
 165         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE"]}]
 166         expectations_before = "BUG_OLD XP RELEASE :  failures/expected/image.html = TEXT"
 167         expectations_after = "BUG_OLD XP RELEASE GPU : failures/expected/image.html = TEXT\nBUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
 168         self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
 169
 170     def test_spurious_updates(self):
 171         failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": ["IMAGE"]}]
 172         expectations_before = "BUG_OLDER MAC LINUX : failures/expected/image.html = IMAGE+TEXT\nBUG_OLD XP RELEASE CPU :  failures/expected/image.html = TEXT"
 173         expectations_after = "BUG_OLDER MAC LINUX : failures/expected/image.html = IMAGE+TEXT\nBUG_NEW XP RELEASE CPU : failures/expected/image.html = IMAGE"
 174         self.assert_update(failure_info_list, expectations_before=expectations_before, expectations_after=expectations_after)
 175
 176
 177 class GardeningServerTest(unittest.TestCase):
 178     def _post_to_path(self, path, body=None, expected_stderr=None, expected_stdout=None):
 179         handler = TestGardeningHTTPRequestHandler(MockServer())
 180         handler.path = path
 181         handler.body = body
 182         OutputCapture().assert_outputs(self, handler.do_POST, expected_stderr=expected_stderr, expected_stdout=expected_stdout)
 183
 184     def test_rollout(self):
 185         expected_stderr = "MOCK run_command: ['echo', 'rollout', '--force-clean', '--non-interactive', '2314', 'MOCK rollout reason'], cwd=/mock-checkout\n"
 186         expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n"
 187         self._post_to_path("/rollout?revision=2314&reason=MOCK+rollout+reason", expected_stderr=expected_stderr, expected_stdout=expected_stdout)
 188
 189     def test_rebaseline(self):
 190         expected_stderr = "MOCK run_command: ['echo', 'rebaseline-test', 'MOCK builder', 'user-scripts/another-test.html'], cwd=/mock-checkout\n"
 191         expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n"
 192         self._post_to_path("/rebaseline?builder=MOCK+builder&test=user-scripts/another-test.html", expected_stderr=expected_stderr, expected_stdout=expected_stdout)
 193
 194     def test_optimizebaselines(self):
 195         expected_stderr = "MOCK run_command: ['echo', 'optimize-baselines', 'user-scripts/another-test.html'], cwd=/mock-checkout\n"
 196         expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n"
 197         self._post_to_path("/optimizebaselines?test=user-scripts/another-test.html", expected_stderr=expected_stderr, expected_stdout=expected_stdout)
 198
 199     def test_updateexpectations(self):
 200         expected_stderr = ""
 201         expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n"
 202         self._post_to_path("/updateexpectations", body="[]", expected_stderr=expected_stderr, expected_stdout=expected_stdout)