Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import StringIO
  30 import sys
  31 import time
  32 import unittest
  33
  34 from webkitpy.common.system import logtesting
  35 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
  36 from webkitpy.common.system.filesystem_mock import MockFileSystem
  37 from webkitpy.common.system.systemhost_mock import MockSystemHost
  38 from webkitpy.layout_tests.port.config_mock import MockConfig
  39 from webkitpy.thirdparty.mock import Mock
  40 from webkitpy.tool.mocktool import MockOptions
  41
  42 import chromium
  43 import chromium_android
  44 import chromium_linux
  45 import chromium_mac
  46 import chromium_win
  47
  48 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  49 from webkitpy.layout_tests.port import port_testcase
  50 from webkitpy.layout_tests.port.driver import DriverInput
  51
  52
  53 class ChromiumDriverTest(unittest.TestCase):
  54     def setUp(self):
  55         host = MockSystemHost()
  56         options = MockOptions(configuration='Release', additional_drt_flag=['--test-shell'])
  57         config = MockConfig(filesystem=host.filesystem, default_configuration='Release')
  58         self.port = chromium_mac.ChromiumMacPort(host, 'chromium-mac-snowleopard', options=options, config=config)
  59         self.driver = chromium.ChromiumDriver(self.port, worker_number=0, pixel_tests=True)
  60
  61     def test_test_shell_command(self):
  62         expected_command = "test.html 2 checksum\n"
  63         self.assertEqual(self.driver._test_shell_command("test.html", 2, "checksum"), expected_command)
  64
  65     def _assert_write_command_and_read_line(self, input=None, expected_line=None, expected_stdin=None, expected_crash=False):
  66         if not expected_stdin:
  67             if input:
  68                 expected_stdin = input
  69             else:
  70                 # We reset stdin, so we should expect stdin.getValue = ""
  71                 expected_stdin = ""
  72         self.driver._proc.stdin = StringIO.StringIO()
  73         line, did_crash = self.driver._write_command_and_read_line(input)
  74         self.assertEqual(self.driver._proc.stdin.getvalue(), expected_stdin)
  75         self.assertEqual(line, expected_line)
  76         self.assertEqual(did_crash, expected_crash)
  77
  78     def test_write_command_and_read_line(self):
  79         self.driver._proc = Mock()  # FIXME: This should use a tighter mock.
  80         # Set up to read 3 lines before we get an IOError
  81         self.driver._proc.stdout = StringIO.StringIO("first\nsecond\nthird\n")
  82
  83         unicode_input = u"I \u2661 Unicode"
  84         utf8_input = unicode_input.encode("utf-8")
  85         # Test unicode input conversion to utf-8
  86         self._assert_write_command_and_read_line(input=unicode_input, expected_stdin=utf8_input, expected_line="first\n")
  87         # Test str() input.
  88         self._assert_write_command_and_read_line(input="foo", expected_line="second\n")
  89         # Test input=None
  90         self._assert_write_command_and_read_line(expected_line="third\n")
  91         # Test reading from a closed/empty stream.
  92         # reading from a StringIO does not raise IOError like a real file would, so raise IOError manually.
  93         def mock_readline():
  94             raise IOError
  95         self.driver._proc.stdout.readline = mock_readline
  96         self._assert_write_command_and_read_line(expected_crash=True)
  97
  98     def test_crash_log(self):
  99         self.driver._proc = Mock()
 100
 101         # Simulate a crash by having stdout close unexpectedly.
 102         def mock_readline():
 103             raise IOError
 104         self.driver._proc.stdout.readline = mock_readline
 105         self.driver._proc.pid = 1234
 106
 107         self.driver.test_to_uri = lambda test: 'mocktesturi'
 108         self.driver._port.driver_name = lambda: 'mockdriver'
 109         self.driver._port._get_crash_log = lambda name, pid, out, err, newer_than: 'mockcrashlog'
 110         driver_output = self.driver.run_test(DriverInput(test_name='some/test.html', timeout=1, image_hash=None, should_run_pixel_test=False))
 111         self.assertTrue(driver_output.crash)
 112         self.assertEqual(driver_output.crashed_process_name, 'mockdriver')
 113         self.assertEqual(driver_output.crashed_pid, 1234)
 114         self.assertEqual(driver_output.crash_log, 'mockcrashlog')
 115
 116     def test_stop(self):
 117         self.pid = None
 118         self.wait_called = False
 119         self.driver._proc = Mock()  # FIXME: This should use a tighter mock.
 120         self.driver._proc.pid = 1
 121         self.driver._proc.stdin = StringIO.StringIO()
 122         self.driver._proc.stdout = StringIO.StringIO()
 123         self.driver._proc.stderr = StringIO.StringIO()
 124         self.driver._proc.poll = lambda: None
 125
 126         def fake_wait():
 127             self.assertTrue(self.pid is not None)
 128             self.wait_called = True
 129
 130         self.driver._proc.wait = fake_wait
 131
 132         class FakeExecutive(object):
 133             def kill_process(other, pid):
 134                 self.pid = pid
 135                 self.driver._proc.poll = lambda: 2
 136
 137         self.driver._port._executive = FakeExecutive()
 138         self.driver.KILL_TIMEOUT_DEFAULT = 0.01
 139         self.driver.stop()
 140         self.assertTrue(self.wait_called)
 141         self.assertEquals(self.pid, 1)
 142
 143     def test_two_drivers(self):
 144
 145         class MockDriver(chromium.ChromiumDriver):
 146             def __init__(self, port):
 147                 chromium.ChromiumDriver.__init__(self, port, worker_number=0, pixel_tests=False)
 148
 149             def cmd_line(self, pixel_test, per_test_args):
 150                 return 'python'
 151
 152         # get_option is used to get the timeout (ms) for a process before we kill it.
 153         driver1 = MockDriver(self.port)
 154         driver1._start(False, [])
 155         driver2 = MockDriver(self.port)
 156         driver2._start(False, [])
 157         # It's possible for driver1 to timeout when stopping if it's sharing stdin with driver2.
 158         start_time = time.time()
 159         driver1.stop()
 160         driver2.stop()
 161         self.assertTrue(time.time() - start_time < 20)
 162
 163
 164 class ChromiumPortTest(port_testcase.PortTestCase):
 165     port_name = 'chromium-mac'
 166     port_maker = chromium.ChromiumPort
 167
 168     def test_all_test_configurations(self):
 169         """Validate the complete set of configurations this port knows about."""
 170         port = self.make_port()
 171         self.assertEquals(set(port.all_test_configurations()), set([
 172             TestConfiguration('icecreamsandwich', 'x86', 'debug'),
 173             TestConfiguration('icecreamsandwich', 'x86', 'release'),
 174             TestConfiguration('leopard', 'x86', 'debug'),
 175             TestConfiguration('leopard', 'x86', 'release'),
 176             TestConfiguration('snowleopard', 'x86', 'debug'),
 177             TestConfiguration('snowleopard', 'x86', 'release'),
 178             TestConfiguration('lion', 'x86', 'debug'),
 179             TestConfiguration('lion', 'x86', 'release'),
 180             TestConfiguration('xp', 'x86', 'debug'),
 181             TestConfiguration('xp', 'x86', 'release'),
 182             TestConfiguration('vista', 'x86', 'debug'),
 183             TestConfiguration('vista', 'x86', 'release'),
 184             TestConfiguration('win7', 'x86', 'debug'),
 185             TestConfiguration('win7', 'x86', 'release'),
 186             TestConfiguration('lucid', 'x86', 'debug'),
 187             TestConfiguration('lucid', 'x86', 'release'),
 188             TestConfiguration('lucid', 'x86_64', 'debug'),
 189             TestConfiguration('lucid', 'x86_64', 'release'),
 190         ]))
 191
 192     def test_driver_cmd_line(self):
 193         # Override this test since ChromiumPort doesn't implement driver_cmd_line().
 194         pass
 195
 196     def test_check_build(self):
 197         # Override this test since ChromiumPort doesn't implement _path_to_driver().
 198         pass
 199
 200     def test_check_wdiff(self):
 201         # Override this test since ChromiumPort doesn't implement _path_to_driver().
 202         pass
 203
 204     class TestMacPort(chromium_mac.ChromiumMacPort):
 205         def __init__(self, options=None):
 206             options = options or MockOptions()
 207             chromium_mac.ChromiumMacPort.__init__(self, MockSystemHost(os_name='mac', os_version='leopard'), 'chromium-mac-leopard', options=options)
 208
 209         def default_configuration(self):
 210             self.default_configuration_called = True
 211             return 'default'
 212
 213     class TestAndroidPort(chromium_android.ChromiumAndroidPort):
 214         def __init__(self, options=None):
 215             options = options or MockOptions()
 216             chromium_win.ChromiumAndroidPort.__init__(self, MockSystemHost(os_name='android', os_version='icecreamsandwich'), 'chromium-android', options=options)
 217
 218         def default_configuration(self):
 219             self.default_configuration_called = True
 220             return 'default'
 221
 222     class TestLinuxPort(chromium_linux.ChromiumLinuxPort):
 223         def __init__(self, options=None):
 224             options = options or MockOptions()
 225             chromium_linux.ChromiumLinuxPort.__init__(self, MockSystemHost(os_name='linux', os_version='lucid'), 'chromium-linux-x86', options=options)
 226
 227         def default_configuration(self):
 228             self.default_configuration_called = True
 229             return 'default'
 230
 231     class TestWinPort(chromium_win.ChromiumWinPort):
 232         def __init__(self, options=None):
 233             options = options or MockOptions()
 234             chromium_win.ChromiumWinPort.__init__(self, MockSystemHost(os_name='win', os_version='xp'), 'chromium-win-xp', options=options)
 235
 236         def default_configuration(self):
 237             self.default_configuration_called = True
 238             return 'default'
 239
 240     def test_path_to_image_diff(self):
 241         # FIXME: These don't need to use endswith now that the port uses a MockFileSystem.
 242         self.assertTrue(ChromiumPortTest.TestLinuxPort()._path_to_image_diff().endswith('/out/default/ImageDiff'))
 243         self.assertTrue(ChromiumPortTest.TestMacPort()._path_to_image_diff().endswith('/xcodebuild/default/ImageDiff'))
 244         self.assertTrue(ChromiumPortTest.TestWinPort()._path_to_image_diff().endswith('/default/ImageDiff.exe'))
 245
 246     def test_default_configuration(self):
 247         mock_options = MockOptions()
 248         port = ChromiumPortTest.TestLinuxPort(options=mock_options)
 249         self.assertEquals(mock_options.configuration, 'default')
 250         self.assertTrue(port.default_configuration_called)
 251
 252         mock_options = MockOptions(configuration=None)
 253         port = ChromiumPortTest.TestLinuxPort(mock_options)
 254         self.assertEquals(mock_options.configuration, 'default')
 255         self.assertTrue(port.default_configuration_called)
 256
 257     def test_diff_image(self):
 258         class TestPort(ChromiumPortTest.TestLinuxPort):
 259             def _path_to_image_diff(self):
 260                 return "/path/to/image_diff"
 261
 262         port = ChromiumPortTest.TestLinuxPort()
 263         mock_image_diff = "MOCK Image Diff"
 264
 265         def mock_run_command(args):
 266             port._filesystem.write_binary_file(args[4], mock_image_diff)
 267             return 1
 268
 269         # Images are different.
 270         port._executive = MockExecutive2(run_command_fn=mock_run_command)
 271         self.assertEquals(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0])
 272
 273         # Images are the same.
 274         port._executive = MockExecutive2(exit_code=0)
 275         self.assertEquals(None, port.diff_image("EXPECTED", "ACTUAL")[0])
 276
 277         # There was some error running image_diff.
 278         port._executive = MockExecutive2(exit_code=2)
 279         exception_raised = False
 280         try:
 281             port.diff_image("EXPECTED", "ACTUAL")
 282         except ValueError, e:
 283             exception_raised = True
 284         self.assertFalse(exception_raised)
 285
 286     def test_overrides_and_builder_names(self):
 287         port = self.make_port()
 288
 289         filesystem = MockFileSystem()
 290         port._filesystem = filesystem
 291         port.path_from_chromium_base = lambda *comps: '/' + '/'.join(comps)
 292
 293         chromium_overrides_path = port.path_from_chromium_base(
 294             'webkit', 'tools', 'layout_tests', 'test_expectations.txt')
 295         CHROMIUM_OVERRIDES = 'contents of %s\n' % chromium_overrides_path
 296         filesystem.write_text_file(chromium_overrides_path, CHROMIUM_OVERRIDES)
 297         skia_overrides_path = port.path_from_chromium_base(
 298             'skia', 'skia_test_expectations.txt')
 299         SKIA_OVERRIDES = 'contents of %s\n' % skia_overrides_path
 300         filesystem.write_text_file(skia_overrides_path, SKIA_OVERRIDES)
 301
 302         additional_expectations_path = port.path_from_chromium_base(
 303             'additional_expectations.txt')
 304         ADDITIONAL_EXPECTATIONS = 'contents of %s\n' % additional_expectations_path
 305         filesystem.write_text_file(additional_expectations_path, ADDITIONAL_EXPECTATIONS)
 306
 307         port._options.builder_name = 'DUMMY_BUILDER_NAME'
 308         port._options.additional_expectations = []
 309         self.assertEquals(port.test_expectations_overrides(),
 310                           SKIA_OVERRIDES + CHROMIUM_OVERRIDES)
 311         port._options.additional_expectations = [additional_expectations_path]
 312         self.assertEquals(port.test_expectations_overrides(),
 313                           SKIA_OVERRIDES + CHROMIUM_OVERRIDES + ADDITIONAL_EXPECTATIONS)
 314
 315         port._options.builder_name = 'builder (deps)'
 316         port._options.additional_expectations = []
 317         self.assertEquals(port.test_expectations_overrides(),
 318                           SKIA_OVERRIDES + CHROMIUM_OVERRIDES)
 319         port._options.additional_expectations = [additional_expectations_path]
 320         self.assertEquals(port.test_expectations_overrides(),
 321                           SKIA_OVERRIDES + CHROMIUM_OVERRIDES + ADDITIONAL_EXPECTATIONS)
 322
 323         # A builder which does NOT observe the Chromium test_expectations,
 324         # but still observes the Skia test_expectations...
 325         port._options.builder_name = 'builder'
 326         port._options.additional_expectations = []
 327         self.assertEquals(port.test_expectations_overrides(),
 328                           SKIA_OVERRIDES)
 329         port._options.additional_expectations = [additional_expectations_path]
 330         self.assertEquals(port.test_expectations_overrides(),
 331                           SKIA_OVERRIDES + ADDITIONAL_EXPECTATIONS)
 332
 333     def test_driver_name_option(self):
 334         self.assertTrue(ChromiumPortTest.TestLinuxPort()._path_to_driver().endswith('/out/default/DumpRenderTree'))
 335         self.assertTrue(ChromiumPortTest.TestMacPort()._path_to_driver().endswith('/xcodebuild/default/DumpRenderTree.app/Contents/MacOS/DumpRenderTree'))
 336         self.assertTrue(ChromiumPortTest.TestWinPort()._path_to_driver().endswith('/default/DumpRenderTree.exe'))
 337
 338         options = MockOptions(driver_name='OtherDriver')
 339         self.assertTrue(ChromiumPortTest.TestLinuxPort(options)._path_to_driver().endswith('/out/default/OtherDriver'))
 340         self.assertTrue(ChromiumPortTest.TestMacPort(options)._path_to_driver().endswith('/xcodebuild/default/OtherDriver.app/Contents/MacOS/OtherDriver'))
 341         self.assertTrue(ChromiumPortTest.TestWinPort(options)._path_to_driver().endswith('/default/OtherDriver.exe'))
 342
 343
 344 class ChromiumPortLoggingTest(logtesting.LoggingTestCase):
 345     def test_check_sys_deps(self):
 346         port = ChromiumPortTest.TestLinuxPort()
 347
 348         # Success
 349         port._executive = MockExecutive2(exit_code=0)
 350         self.assertTrue(port.check_sys_deps(needs_http=False))
 351
 352         # Failure
 353         port._executive = MockExecutive2(exit_code=1,
 354             output='testing output failure')
 355         self.assertFalse(port.check_sys_deps(needs_http=False))
 356         self.assertLog([
 357             'ERROR: System dependencies check failed.\n',
 358             'ERROR: To override, invoke with --nocheck-sys-deps\n',
 359             'ERROR: \n',
 360             'ERROR: testing output failure\n'])
 361
 362
 363 if __name__ == '__main__':
 364     unittest.main()