Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import StringIO
  30 import sys
  31 import time
  32 import unittest
  33
  34 from webkitpy.common.system import logtesting
  35 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
  36 from webkitpy.common.system.filesystem_mock import MockFileSystem
  37 from webkitpy.common.host_mock import MockHost
  38 from webkitpy.thirdparty.mock import Mock
  39 from webkitpy.tool.mocktool import MockOptions
  40
  41 import chromium
  42 import chromium_linux
  43 import chromium_mac
  44 import chromium_win
  45
  46 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  47 from webkitpy.layout_tests.port import port_testcase
  48 from webkitpy.layout_tests.port.driver import DriverInput
  49
  50
  51 class ChromiumDriverTest(unittest.TestCase):
  52     def setUp(self):
  53         mock_port = Mock()  # FIXME: This should use a tighter mock.
  54         self.driver = chromium.ChromiumDriver(mock_port, worker_number=0, pixel_tests=True)
  55
  56     def test_test_shell_command(self):
  57         expected_command = "test.html 2 checksum\n"
  58         self.assertEqual(self.driver._test_shell_command("test.html", 2, "checksum"), expected_command)
  59
  60     def _assert_write_command_and_read_line(self, input=None, expected_line=None, expected_stdin=None, expected_crash=False):
  61         if not expected_stdin:
  62             if input:
  63                 expected_stdin = input
  64             else:
  65                 # We reset stdin, so we should expect stdin.getValue = ""
  66                 expected_stdin = ""
  67         self.driver._proc.stdin = StringIO.StringIO()
  68         line, did_crash = self.driver._write_command_and_read_line(input)
  69         self.assertEqual(self.driver._proc.stdin.getvalue(), expected_stdin)
  70         self.assertEqual(line, expected_line)
  71         self.assertEqual(did_crash, expected_crash)
  72
  73     def test_write_command_and_read_line(self):
  74         self.driver._proc = Mock()  # FIXME: This should use a tighter mock.
  75         # Set up to read 3 lines before we get an IOError
  76         self.driver._proc.stdout = StringIO.StringIO("first\nsecond\nthird\n")
  77
  78         unicode_input = u"I \u2661 Unicode"
  79         utf8_input = unicode_input.encode("utf-8")
  80         # Test unicode input conversion to utf-8
  81         self._assert_write_command_and_read_line(input=unicode_input, expected_stdin=utf8_input, expected_line="first\n")
  82         # Test str() input.
  83         self._assert_write_command_and_read_line(input="foo", expected_line="second\n")
  84         # Test input=None
  85         self._assert_write_command_and_read_line(expected_line="third\n")
  86         # Test reading from a closed/empty stream.
  87         # reading from a StringIO does not raise IOError like a real file would, so raise IOError manually.
  88         def mock_readline():
  89             raise IOError
  90         self.driver._proc.stdout.readline = mock_readline
  91         self._assert_write_command_and_read_line(expected_crash=True)
  92
  93     def test_crashed_process_name(self):
  94         self.driver._proc = Mock()
  95
  96         # Simulate a crash by having stdout close unexpectedly.
  97         def mock_readline():
  98             raise IOError
  99         self.driver._proc.stdout.readline = mock_readline
 100
 101         self.driver._port.test_to_uri = lambda test: 'mocktesturi'
 102         driver_output = self.driver.run_test(DriverInput(test_name='some/test.html', timeout=1, image_hash=None, is_reftest=False))
 103         self.assertEqual(self.driver._port.driver_name(), driver_output.crashed_process_name)
 104
 105     def test_stop(self):
 106         self.pid = None
 107         self.wait_called = False
 108         self.driver._proc = Mock()  # FIXME: This should use a tighter mock.
 109         self.driver._proc.pid = 1
 110         self.driver._proc.stdin = StringIO.StringIO()
 111         self.driver._proc.stdout = StringIO.StringIO()
 112         self.driver._proc.stderr = StringIO.StringIO()
 113         self.driver._proc.poll = lambda: None
 114
 115         def fake_wait():
 116             self.assertTrue(self.pid is not None)
 117             self.wait_called = True
 118
 119         self.driver._proc.wait = fake_wait
 120
 121         class FakeExecutive(object):
 122             def kill_process(other, pid):
 123                 self.pid = pid
 124                 self.driver._proc.poll = lambda: 2
 125
 126         self.driver._port._executive = FakeExecutive()
 127         # Override the kill timeout (ms) so the test runs faster.
 128         self.driver._port.get_option = lambda name: 1
 129         self.driver.stop()
 130         self.assertTrue(self.wait_called)
 131         self.assertEquals(self.pid, 1)
 132
 133     def test_two_drivers(self):
 134         mock_port = Mock()
 135
 136         class MockDriver(chromium.ChromiumDriver):
 137             def __init__(self):
 138                 chromium.ChromiumDriver.__init__(self, mock_port, worker_number=0, pixel_tests=False)
 139
 140             def cmd_line(self):
 141                 return 'python'
 142
 143         # get_option is used to get the timeout (ms) for a process before we kill it.
 144         mock_port.get_option = lambda name: 60 * 1000
 145         driver1 = MockDriver()
 146         driver1._start()
 147         driver2 = MockDriver()
 148         driver2._start()
 149         # It's possible for driver1 to timeout when stopping if it's sharing stdin with driver2.
 150         start_time = time.time()
 151         driver1.stop()
 152         driver2.stop()
 153         self.assertTrue(time.time() - start_time < 20)
 154
 155
 156 class ChromiumPortTest(port_testcase.PortTestCase):
 157     port_maker = chromium.ChromiumPort
 158
 159     def test_all_test_configurations(self):
 160         """Validate the complete set of configurations this port knows about."""
 161         port = chromium.ChromiumPort(MockHost())
 162         self.assertEquals(set(port.all_test_configurations()), set([
 163             TestConfiguration('leopard', 'x86', 'debug', 'cpu'),
 164             TestConfiguration('leopard', 'x86', 'debug', 'gpu'),
 165             TestConfiguration('leopard', 'x86', 'debug', 'cpu-cg'),
 166             TestConfiguration('leopard', 'x86', 'debug', 'gpu-cg'),
 167             TestConfiguration('leopard', 'x86', 'release', 'cpu'),
 168             TestConfiguration('leopard', 'x86', 'release', 'gpu'),
 169             TestConfiguration('leopard', 'x86', 'release', 'cpu-cg'),
 170             TestConfiguration('leopard', 'x86', 'release', 'gpu-cg'),
 171             TestConfiguration('snowleopard', 'x86', 'debug', 'cpu'),
 172             TestConfiguration('snowleopard', 'x86', 'debug', 'gpu'),
 173             TestConfiguration('snowleopard', 'x86', 'debug', 'cpu-cg'),
 174             TestConfiguration('snowleopard', 'x86', 'debug', 'gpu-cg'),
 175             TestConfiguration('snowleopard', 'x86', 'release', 'cpu'),
 176             TestConfiguration('snowleopard', 'x86', 'release', 'gpu'),
 177             TestConfiguration('snowleopard', 'x86', 'release', 'cpu-cg'),
 178             TestConfiguration('snowleopard', 'x86', 'release', 'gpu-cg'),
 179             TestConfiguration('lion', 'x86', 'debug', 'cpu'),
 180             TestConfiguration('lion', 'x86', 'debug', 'gpu'),
 181             TestConfiguration('lion', 'x86', 'debug', 'cpu-cg'),
 182             TestConfiguration('lion', 'x86', 'debug', 'gpu-cg'),
 183             TestConfiguration('lion', 'x86', 'release', 'cpu'),
 184             TestConfiguration('lion', 'x86', 'release', 'gpu'),
 185             TestConfiguration('lion', 'x86', 'release', 'cpu-cg'),
 186             TestConfiguration('lion', 'x86', 'release', 'gpu-cg'),
 187             TestConfiguration('xp', 'x86', 'debug', 'cpu'),
 188             TestConfiguration('xp', 'x86', 'debug', 'gpu'),
 189             TestConfiguration('xp', 'x86', 'release', 'cpu'),
 190             TestConfiguration('xp', 'x86', 'release', 'gpu'),
 191             TestConfiguration('vista', 'x86', 'debug', 'cpu'),
 192             TestConfiguration('vista', 'x86', 'debug', 'gpu'),
 193             TestConfiguration('vista', 'x86', 'release', 'cpu'),
 194             TestConfiguration('vista', 'x86', 'release', 'gpu'),
 195             TestConfiguration('win7', 'x86', 'debug', 'cpu'),
 196             TestConfiguration('win7', 'x86', 'debug', 'gpu'),
 197             TestConfiguration('win7', 'x86', 'release', 'cpu'),
 198             TestConfiguration('win7', 'x86', 'release', 'gpu'),
 199             TestConfiguration('lucid', 'x86', 'debug', 'cpu'),
 200             TestConfiguration('lucid', 'x86', 'debug', 'gpu'),
 201             TestConfiguration('lucid', 'x86', 'release', 'cpu'),
 202             TestConfiguration('lucid', 'x86', 'release', 'gpu'),
 203             TestConfiguration('lucid', 'x86_64', 'debug', 'cpu'),
 204             TestConfiguration('lucid', 'x86_64', 'debug', 'gpu'),
 205             TestConfiguration('lucid', 'x86_64', 'release', 'cpu'),
 206             TestConfiguration('lucid', 'x86_64', 'release', 'gpu'),
 207         ]))
 208
 209     def test_driver_cmd_line(self):
 210         # Override this test since ChromiumPort doesn't implement driver_cmd_line().
 211         pass
 212
 213     def test_check_build(self):
 214         # Override this test since ChromiumPort doesn't implement _path_to_driver().
 215         pass
 216
 217     def test_check_wdiff(self):
 218         # Override this test since ChromiumPort doesn't implement _path_to_driver().
 219         pass
 220
 221     class TestMacPort(chromium_mac.ChromiumMacPort):
 222         def __init__(self, options=None):
 223             options = options or MockOptions()
 224             chromium_mac.ChromiumMacPort.__init__(self, MockHost(), options=options)
 225
 226         def default_configuration(self):
 227             self.default_configuration_called = True
 228             return 'default'
 229
 230     class TestLinuxPort(chromium_linux.ChromiumLinuxPort):
 231         def __init__(self, options=None):
 232             options = options or MockOptions()
 233             chromium_linux.ChromiumLinuxPort.__init__(self, MockHost(), options=options)
 234
 235         def default_configuration(self):
 236             self.default_configuration_called = True
 237             return 'default'
 238
 239     class TestWinPort(chromium_win.ChromiumWinPort):
 240         def __init__(self, options=None):
 241             options = options or MockOptions()
 242             chromium_win.ChromiumWinPort.__init__(self, MockHost(), options=options)
 243
 244         def default_configuration(self):
 245             self.default_configuration_called = True
 246             return 'default'
 247
 248     def test_path_to_image_diff(self):
 249         # FIXME: These don't need to use endswith now that the port uses a MockFileSystem.
 250         self.assertTrue(ChromiumPortTest.TestLinuxPort()._path_to_image_diff().endswith('/out/default/ImageDiff'))
 251         self.assertTrue(ChromiumPortTest.TestMacPort()._path_to_image_diff().endswith('/xcodebuild/default/ImageDiff'))
 252         self.assertTrue(ChromiumPortTest.TestWinPort()._path_to_image_diff().endswith('/default/ImageDiff.exe'))
 253
 254     def test_skipped_layout_tests(self):
 255         mock_options = MockOptions()
 256         mock_options.configuration = 'release'
 257         port = ChromiumPortTest.TestLinuxPort(options=mock_options)
 258
 259         fake_test = 'fast/js/not-good.js'
 260
 261         port.test_expectations = lambda: """BUG_TEST SKIP : fast/js/not-good.js = TEXT
 262 LINUX WIN : fast/js/very-good.js = TIMEOUT PASS"""
 263         port.test_expectations_overrides = lambda: ''
 264         port.tests = lambda paths: set()
 265         port.test_exists = lambda test: True
 266
 267         skipped_tests = port.skipped_layout_tests(extra_test_files=[fake_test, ])
 268         self.assertTrue("fast/js/not-good.js" in skipped_tests)
 269
 270     def test_default_configuration(self):
 271         mock_options = MockOptions()
 272         port = ChromiumPortTest.TestLinuxPort(options=mock_options)
 273         self.assertEquals(mock_options.configuration, 'default')
 274         self.assertTrue(port.default_configuration_called)
 275
 276         mock_options = MockOptions(configuration=None)
 277         port = ChromiumPortTest.TestLinuxPort(mock_options)
 278         self.assertEquals(mock_options.configuration, 'default')
 279         self.assertTrue(port.default_configuration_called)
 280
 281     def test_diff_image(self):
 282         class TestPort(ChromiumPortTest.TestLinuxPort):
 283             def _path_to_image_diff(self):
 284                 return "/path/to/image_diff"
 285
 286         port = ChromiumPortTest.TestLinuxPort()
 287         mock_image_diff = "MOCK Image Diff"
 288
 289         def mock_run_command(args):
 290             port._filesystem.write_binary_file(args[4], mock_image_diff)
 291             return 1
 292
 293         # Images are different.
 294         port._executive = MockExecutive2(run_command_fn=mock_run_command)
 295         self.assertEquals(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0])
 296
 297         # Images are the same.
 298         port._executive = MockExecutive2(exit_code=0)
 299         self.assertEquals(None, port.diff_image("EXPECTED", "ACTUAL")[0])
 300
 301         # There was some error running image_diff.
 302         port._executive = MockExecutive2(exit_code=2)
 303         exception_raised = False
 304         try:
 305             port.diff_image("EXPECTED", "ACTUAL")
 306         except ValueError, e:
 307             exception_raised = True
 308         self.assertFalse(exception_raised)
 309
 310     def test_overrides_and_builder_names(self):
 311         port = self.make_port()
 312
 313         filesystem = MockFileSystem()
 314         port._filesystem = filesystem
 315         port.path_from_chromium_base = lambda *comps: '/' + '/'.join(comps)
 316
 317         overrides_path = port.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations.txt')
 318         OVERRIDES = 'foo'
 319         filesystem.files[overrides_path] = OVERRIDES
 320
 321         port._options.builder_name = 'DUMMY_BUILDER_NAME'
 322         self.assertEquals(port.test_expectations_overrides(), OVERRIDES)
 323
 324         port._options.builder_name = 'builder (deps)'
 325         self.assertEquals(port.test_expectations_overrides(), OVERRIDES)
 326
 327         port._options.builder_name = 'builder'
 328         self.assertEquals(port.test_expectations_overrides(), None)
 329
 330
 331 class ChromiumPortLoggingTest(logtesting.LoggingTestCase):
 332     def test_check_sys_deps(self):
 333         port = ChromiumPortTest.TestLinuxPort()
 334
 335         # Success
 336         port._executive = MockExecutive2(exit_code=0)
 337         self.assertTrue(port.check_sys_deps(needs_http=False))
 338
 339         # Failure
 340         port._executive = MockExecutive2(exit_code=1,
 341             output='testing output failure')
 342         self.assertFalse(port.check_sys_deps(needs_http=False))
 343         self.assertLog([
 344             'ERROR: System dependencies check failed.\n',
 345             'ERROR: To override, invoke with --nocheck-sys-deps\n',
 346             'ERROR: \n',
 347             'ERROR: testing output failure\n'])
 348
 349
 350 if __name__ == '__main__':
 351     unittest.main()