src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 """Unit testing base class for Port implementations."""
  30
  31 import errno
  32 import logging
  33 import os
  34 import socket
  35 import sys
  36 import time
  37 import webkitpy.thirdparty.unittest2 as unittest
  38
  39 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
  40 from webkitpy.common.system.filesystem_mock import MockFileSystem
  41 from webkitpy.common.system.outputcapture import OutputCapture
  42 from webkitpy.common.system.systemhost_mock import MockSystemHost
  43 from webkitpy.layout_tests.models import test_run_results
  44 from webkitpy.layout_tests.port.base import Port, TestConfiguration
  45 from webkitpy.layout_tests.port.server_process_mock import MockServerProcess
  46 from webkitpy.tool.mocktool import MockOptions
  47
  48
  49 # FIXME: get rid of this fixture
  50 class TestWebKitPort(Port):
  51     port_name = "testwebkitport"
  52
  53     def __init__(self, port_name=None, symbols_string=None,
  54                  expectations_file=None, skips_file=None, host=None, config=None,
  55                  **kwargs):
  56         port_name = port_name or TestWebKitPort.port_name
  57         self.symbols_string = symbols_string  # Passing "" disables all staticly-detectable features.
  58         host = host or MockSystemHost()
  59         super(TestWebKitPort, self).__init__(host, port_name=port_name, **kwargs)
  60
  61     def all_test_configurations(self):
  62         return [self.test_configuration()]
  63
  64     def _symbols_string(self):
  65         return self.symbols_string
  66
  67     def _tests_for_disabled_features(self):
  68         return ["accessibility", ]
  69
  70
  71 class FakePrinter(object):
  72     def write_update(self, msg):
  73         pass
  74
  75     def write_throttled_update(self, msg):
  76         pass
  77
  78
  79
  80 class PortTestCase(unittest.TestCase):
  81     """Tests that all Port implementations must pass."""
  82     HTTP_PORTS = (8000, 8080, 8443)
  83     WEBSOCKET_PORTS = (8880,)
  84
  85     # Subclasses override this to point to their Port subclass.
  86     os_name = None
  87     os_version = None
  88     port_maker = TestWebKitPort
  89     port_name = None
  90
  91     def make_port(self, host=None, port_name=None, options=None, os_name=None, os_version=None, **kwargs):
  92         host = host or MockSystemHost(os_name=(os_name or self.os_name), os_version=(os_version or self.os_version))
  93         options = options or MockOptions(configuration='Release')
  94         port_name = port_name or self.port_name
  95         port_name = self.port_maker.determine_full_port_name(host, options, port_name)
  96         port = self.port_maker(host, port_name, options=options, **kwargs)
  97         port._config.build_directory = lambda configuration: '/mock-build'
  98         return port
  99
 100     def make_wdiff_available(self, port):
 101         port._wdiff_available = True
 102
 103     def test_check_build(self):
 104         port = self.make_port()
 105         port._check_file_exists = lambda path, desc: True
 106         if port._dump_reader:
 107             port._dump_reader.check_is_functional = lambda: True
 108         port._options.build = True
 109         port._check_driver_build_up_to_date = lambda config: True
 110         oc = OutputCapture()
 111         try:
 112             oc.capture_output()
 113             self.assertEqual(port.check_build(needs_http=True, printer=FakePrinter()),
 114                              test_run_results.OK_EXIT_STATUS)
 115         finally:
 116             out, err, logs = oc.restore_output()
 117             self.assertIn('pretty patches', logs)         # We should get a warning about PrettyPatch being missing,
 118             self.assertNotIn('build requirements', logs)  # but not the driver itself.
 119
 120         port._check_file_exists = lambda path, desc: False
 121         port._check_driver_build_up_to_date = lambda config: False
 122         try:
 123             oc.capture_output()
 124             self.assertEqual(port.check_build(needs_http=True, printer=FakePrinter()),
 125                             test_run_results.UNEXPECTED_ERROR_EXIT_STATUS)
 126         finally:
 127             out, err, logs = oc.restore_output()
 128             self.assertIn('pretty patches', logs)        # And, hereere we should get warnings about both.
 129             self.assertIn('build requirements', logs)
 130
 131     def test_default_max_locked_shards(self):
 132         port = self.make_port()
 133         port.default_child_processes = lambda: 16
 134         self.assertEqual(port.default_max_locked_shards(), 4)
 135         port.default_child_processes = lambda: 2
 136         self.assertEqual(port.default_max_locked_shards(), 1)
 137
 138     def test_default_timeout_ms(self):
 139         self.assertEqual(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 6000)
 140         self.assertEqual(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 18000)
 141
 142     def test_default_pixel_tests(self):
 143         self.assertEqual(self.make_port().default_pixel_tests(), True)
 144
 145     def test_driver_cmd_line(self):
 146         port = self.make_port()
 147         self.assertTrue(len(port.driver_cmd_line()))
 148
 149         options = MockOptions(additional_drt_flag=['--foo=bar', '--foo=baz'])
 150         port = self.make_port(options=options)
 151         cmd_line = port.driver_cmd_line()
 152         self.assertTrue('--foo=bar' in cmd_line)
 153         self.assertTrue('--foo=baz' in cmd_line)
 154
 155     def test_uses_apache(self):
 156         self.assertTrue(self.make_port().uses_apache())
 157
 158     def assert_servers_are_down(self, host, ports):
 159         for port in ports:
 160             try:
 161                 test_socket = socket.socket()
 162                 test_socket.connect((host, port))
 163                 self.fail()
 164             except IOError, e:
 165                 self.assertTrue(e.errno in (errno.ECONNREFUSED, errno.ECONNRESET))
 166             finally:
 167                 test_socket.close()
 168
 169     def assert_servers_are_up(self, host, ports):
 170         for port in ports:
 171             try:
 172                 test_socket = socket.socket()
 173                 test_socket.connect((host, port))
 174             except IOError, e:
 175                 self.fail('failed to connect to %s:%d' % (host, port))
 176             finally:
 177                 test_socket.close()
 178
 179     def test_diff_image__missing_both(self):
 180         port = self.make_port()
 181         self.assertEqual(port.diff_image(None, None), (None, None))
 182         self.assertEqual(port.diff_image(None, ''), (None, None))
 183         self.assertEqual(port.diff_image('', None), (None, None))
 184
 185         self.assertEqual(port.diff_image('', ''), (None, None))
 186
 187     def test_diff_image__missing_actual(self):
 188         port = self.make_port()
 189         self.assertEqual(port.diff_image(None, 'foo'), ('foo', None))
 190         self.assertEqual(port.diff_image('', 'foo'), ('foo', None))
 191
 192     def test_diff_image__missing_expected(self):
 193         port = self.make_port()
 194         self.assertEqual(port.diff_image('foo', None), ('foo', None))
 195         self.assertEqual(port.diff_image('foo', ''), ('foo', None))
 196
 197     def test_diff_image(self):
 198         def _path_to_image_diff():
 199             return "/path/to/image_diff"
 200
 201         port = self.make_port()
 202         port._path_to_image_diff = _path_to_image_diff
 203
 204         mock_image_diff = "MOCK Image Diff"
 205
 206         def mock_run_command(args):
 207             port._filesystem.write_binary_file(args[4], mock_image_diff)
 208             return 1
 209
 210         # Images are different.
 211         port._executive = MockExecutive2(run_command_fn=mock_run_command)
 212         self.assertEqual(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0])
 213
 214         # Images are the same.
 215         port._executive = MockExecutive2(exit_code=0)
 216         self.assertEqual(None, port.diff_image("EXPECTED", "ACTUAL")[0])
 217
 218         # There was some error running image_diff.
 219         port._executive = MockExecutive2(exit_code=2)
 220         exception_raised = False
 221         try:
 222             port.diff_image("EXPECTED", "ACTUAL")
 223         except ValueError, e:
 224             exception_raised = True
 225         self.assertFalse(exception_raised)
 226
 227     def test_diff_image_crashed(self):
 228         port = self.make_port()
 229         port._executive = MockExecutive2(exit_code=2)
 230         self.assertEqual(port.diff_image("EXPECTED", "ACTUAL"), (None, 'image diff returned an exit code of 2'))
 231
 232     def test_check_wdiff(self):
 233         port = self.make_port()
 234         port.check_wdiff()
 235
 236     def test_wdiff_text_fails(self):
 237         host = MockSystemHost(os_name=self.os_name, os_version=self.os_version)
 238         host.executive = MockExecutive(should_throw=True)
 239         port = self.make_port(host=host)
 240         port._executive = host.executive  # AndroidPortTest.make_port sets its own executive, so reset that as well.
 241
 242         # This should raise a ScriptError that gets caught and turned into the
 243         # error text, and also mark wdiff as not available.
 244         self.make_wdiff_available(port)
 245         self.assertTrue(port.wdiff_available())
 246         diff_txt = port.wdiff_text("/tmp/foo.html", "/tmp/bar.html")
 247         self.assertEqual(diff_txt, port._wdiff_error_html)
 248         self.assertFalse(port.wdiff_available())
 249
 250     def test_missing_symbol_to_skipped_tests(self):
 251         # Test that we get the chromium skips and not the webkit default skips
 252         port = self.make_port()
 253         skip_dict = port._missing_symbol_to_skipped_tests()
 254         if port.PORT_HAS_AUDIO_CODECS_BUILT_IN:
 255             self.assertEqual(skip_dict, {})
 256         else:
 257             self.assertTrue('ff_mp3_decoder' in skip_dict)
 258         self.assertFalse('WebGLShader' in skip_dict)
 259
 260     def test_test_configuration(self):
 261         port = self.make_port()
 262         self.assertTrue(port.test_configuration())
 263
 264     def test_all_test_configurations(self):
 265         """Validate the complete set of configurations this port knows about."""
 266         port = self.make_port()
 267         self.assertEqual(set(port.all_test_configurations()), set([
 268             TestConfiguration('snowleopard', 'x86', 'debug'),
 269             TestConfiguration('snowleopard', 'x86', 'release'),
 270             TestConfiguration('lion', 'x86', 'debug'),
 271             TestConfiguration('lion', 'x86', 'release'),
 272             TestConfiguration('retina', 'x86', 'debug'),
 273             TestConfiguration('retina', 'x86', 'release'),
 274             TestConfiguration('mountainlion', 'x86', 'debug'),
 275             TestConfiguration('mountainlion', 'x86', 'release'),
 276             TestConfiguration('mavericks', 'x86', 'debug'),
 277             TestConfiguration('mavericks', 'x86', 'release'),
 278             TestConfiguration('xp', 'x86', 'debug'),
 279             TestConfiguration('xp', 'x86', 'release'),
 280             TestConfiguration('win7', 'x86', 'debug'),
 281             TestConfiguration('win7', 'x86', 'release'),
 282             TestConfiguration('lucid', 'x86', 'debug'),
 283             TestConfiguration('lucid', 'x86', 'release'),
 284             TestConfiguration('lucid', 'x86_64', 'debug'),
 285             TestConfiguration('lucid', 'x86_64', 'release'),
 286             TestConfiguration('icecreamsandwich', 'x86', 'debug'),
 287             TestConfiguration('icecreamsandwich', 'x86', 'release'),
 288         ]))
 289     def test_get_crash_log(self):
 290         port = self.make_port()
 291         self.assertEqual(port._get_crash_log(None, None, None, None, newer_than=None),
 292            (None,
 293             'crash log for <unknown process name> (pid <unknown>):\n'
 294             'STDOUT: <empty>\n'
 295             'STDERR: <empty>\n'))
 296
 297         self.assertEqual(port._get_crash_log('foo', 1234, 'out bar\nout baz', 'err bar\nerr baz\n', newer_than=None),
 298             ('err bar\nerr baz\n',
 299              'crash log for foo (pid 1234):\n'
 300              'STDOUT: out bar\n'
 301              'STDOUT: out baz\n'
 302              'STDERR: err bar\n'
 303              'STDERR: err baz\n'))
 304
 305         self.assertEqual(port._get_crash_log('foo', 1234, 'foo\xa6bar', 'foo\xa6bar', newer_than=None),
 306             ('foo\xa6bar',
 307              u'crash log for foo (pid 1234):\n'
 308              u'STDOUT: foo\ufffdbar\n'
 309              u'STDERR: foo\ufffdbar\n'))
 310
 311         self.assertEqual(port._get_crash_log('foo', 1234, 'foo\xa6bar', 'foo\xa6bar', newer_than=1.0),
 312             ('foo\xa6bar',
 313              u'crash log for foo (pid 1234):\n'
 314              u'STDOUT: foo\ufffdbar\n'
 315              u'STDERR: foo\ufffdbar\n'))
 316
 317     def assert_build_path(self, options, dirs, expected_path):
 318         port = self.make_port(options=options)
 319         for directory in dirs:
 320             port.host.filesystem.maybe_make_directory(directory)
 321         self.assertEqual(port._build_path(), expected_path)
 322
 323     def test_expectations_files(self):
 324         port = self.make_port()
 325
 326         generic_path = port.path_to_generic_test_expectations_file()
 327         chromium_overrides_path = port.path_from_chromium_base(
 328             'webkit', 'tools', 'layout_tests', 'test_expectations.txt')
 329         never_fix_tests_path = port._filesystem.join(port.layout_tests_dir(), 'NeverFixTests')
 330         stale_tests_path = port._filesystem.join(port.layout_tests_dir(), 'StaleTestExpectations')
 331         slow_tests_path = port._filesystem.join(port.layout_tests_dir(), 'SlowTests')
 332         skia_overrides_path = port.path_from_chromium_base(
 333             'skia', 'skia_test_expectations.txt')
 334
 335         port._filesystem.write_text_file(skia_overrides_path, 'dummy text')
 336
 337         w3c_overrides_path = port.path_from_chromium_base(
 338             'webkit', 'tools', 'layout_tests', 'test_expectations_w3c.txt')
 339         port._filesystem.write_text_file(w3c_overrides_path, 'dummy text')
 340
 341         port._options.builder_name = 'DUMMY_BUILDER_NAME'
 342         self.assertEqual(port.expectations_files(),
 343                          [generic_path, skia_overrides_path, w3c_overrides_path,
 344                           never_fix_tests_path, stale_tests_path, slow_tests_path,
 345                           chromium_overrides_path])
 346
 347         port._options.builder_name = 'builder (deps)'
 348         self.assertEqual(port.expectations_files(),
 349                          [generic_path, skia_overrides_path, w3c_overrides_path,
 350                           never_fix_tests_path, stale_tests_path, slow_tests_path,
 351                           chromium_overrides_path])
 352
 353         # A builder which does NOT observe the Chromium test_expectations,
 354         # but still observes the Skia test_expectations...
 355         port._options.builder_name = 'builder'
 356         self.assertEqual(port.expectations_files(),
 357                          [generic_path, skia_overrides_path, w3c_overrides_path,
 358                           never_fix_tests_path, stale_tests_path, slow_tests_path])
 359
 360     def test_check_sys_deps(self):
 361         port = self.make_port()
 362         port._executive = MockExecutive2(exit_code=0)
 363         self.assertEqual(port.check_sys_deps(needs_http=False), test_run_results.OK_EXIT_STATUS)
 364         port._executive = MockExecutive2(exit_code=1, output='testing output failure')
 365         self.assertEqual(port.check_sys_deps(needs_http=False), test_run_results.SYS_DEPS_EXIT_STATUS)
 366
 367     def test_expectations_ordering(self):
 368         port = self.make_port()
 369         for path in port.expectations_files():
 370             port._filesystem.write_text_file(path, '')
 371         ordered_dict = port.expectations_dict()
 372         self.assertEqual(port.path_to_generic_test_expectations_file(), ordered_dict.keys()[0])
 373
 374         options = MockOptions(additional_expectations=['/tmp/foo', '/tmp/bar'])
 375         port = self.make_port(options=options)
 376         for path in port.expectations_files():
 377             port._filesystem.write_text_file(path, '')
 378         port._filesystem.write_text_file('/tmp/foo', 'foo')
 379         port._filesystem.write_text_file('/tmp/bar', 'bar')
 380         ordered_dict = port.expectations_dict()
 381         self.assertEqual(ordered_dict.keys()[-2:], options.additional_expectations)  # pylint: disable=E1101
 382         self.assertEqual(ordered_dict.values()[-2:], ['foo', 'bar'])
 383
 384     def test_skipped_directories_for_symbols(self):
 385         # This first test confirms that the commonly found symbols result in the expected skipped directories.
 386         symbols_string = " ".join(["fooSymbol"])
 387         expected_directories = set([
 388             "webaudio/codec-tests/mp3",
 389             "webaudio/codec-tests/aac",
 390         ])
 391
 392         result_directories = set(TestWebKitPort(symbols_string=symbols_string)._skipped_tests_for_unsupported_features(test_list=['webaudio/codec-tests/mp3/foo.html']))
 393         self.assertEqual(result_directories, expected_directories)
 394
 395         # Test that the nm string parsing actually works:
 396         symbols_string = """
 397 000000000124f498 s __ZZN7WebCore13ff_mp3_decoder12replaceChildEPS0_S1_E19__PRETTY_FUNCTION__
 398 000000000124f500 s __ZZN7WebCore13ff_mp3_decoder13addChildAboveEPS0_S1_E19__PRETTY_FUNCTION__
 399 000000000124f670 s __ZZN7WebCore13ff_mp3_decoder13addChildBelowEPS0_S1_E19__PRETTY_FUNCTION__
 400 """
 401         # Note 'compositing' is not in the list of skipped directories (hence the parsing of GraphicsLayer worked):
 402         expected_directories = set([
 403             "webaudio/codec-tests/aac",
 404         ])
 405         result_directories = set(TestWebKitPort(symbols_string=symbols_string)._skipped_tests_for_unsupported_features(test_list=['webaudio/codec-tests/mp3/foo.html']))
 406         self.assertEqual(result_directories, expected_directories)
 407
 408     def _assert_config_file_for_platform(self, port, platform, config_file):
 409         self.assertEqual(port._apache_config_file_name_for_platform(platform), config_file)
 410
 411     def test_linux_distro_detection(self):
 412         port = TestWebKitPort()
 413         self.assertFalse(port._is_redhat_based())
 414         self.assertFalse(port._is_debian_based())
 415
 416         port._filesystem = MockFileSystem({'/etc/redhat-release': ''})
 417         self.assertTrue(port._is_redhat_based())
 418         self.assertFalse(port._is_debian_based())
 419
 420         port._filesystem = MockFileSystem({'/etc/debian_version': ''})
 421         self.assertFalse(port._is_redhat_based())
 422         self.assertTrue(port._is_debian_based())
 423
 424     def test_apache_config_file_name_for_platform(self):
 425         port = TestWebKitPort()
 426         self._assert_config_file_for_platform(port, 'cygwin', 'cygwin-httpd.conf')
 427
 428         self._assert_config_file_for_platform(port, 'linux2', 'apache2-httpd.conf')
 429         self._assert_config_file_for_platform(port, 'linux3', 'apache2-httpd.conf')
 430
 431         port._is_redhat_based = lambda: True
 432         port._apache_version = lambda: '2.2'
 433         self._assert_config_file_for_platform(port, 'linux2', 'fedora-httpd-2.2.conf')
 434
 435         port = TestWebKitPort()
 436         port._is_debian_based = lambda: True
 437         port._apache_version = lambda: '2.2'
 438         self._assert_config_file_for_platform(port, 'linux2', 'debian-httpd-2.2.conf')
 439
 440         self._assert_config_file_for_platform(port, 'mac', 'apache2-httpd.conf')
 441         self._assert_config_file_for_platform(port, 'win32', 'apache2-httpd.conf')  # win32 isn't a supported sys.platform.  AppleWin/WinCairo/WinCE ports all use cygwin.
 442         self._assert_config_file_for_platform(port, 'barf', 'apache2-httpd.conf')
 443
 444     def test_path_to_apache_config_file(self):
 445         port = TestWebKitPort()
 446
 447         saved_environ = os.environ.copy()
 448         try:
 449             os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/path/to/httpd.conf'
 450             self.assertRaises(IOError, port.path_to_apache_config_file)
 451             port._filesystem.write_text_file('/existing/httpd.conf', 'Hello, world!')
 452             os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf'
 453             self.assertEqual(port.path_to_apache_config_file(), '/existing/httpd.conf')
 454         finally:
 455             os.environ = saved_environ.copy()
 456
 457         # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value.
 458         port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf'
 459         self.assertEqual(port.path_to_apache_config_file(), '/mock-checkout/third_party/WebKit/LayoutTests/http/conf/httpd.conf')
 460
 461         # Check that even if we mock out _apache_config_file_name, the environment variable takes precedence.
 462         saved_environ = os.environ.copy()
 463         try:
 464             os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf'
 465             self.assertEqual(port.path_to_apache_config_file(), '/existing/httpd.conf')
 466         finally:
 467             os.environ = saved_environ.copy()
 468
 469     def test_additional_platform_directory(self):
 470         port = self.make_port(options=MockOptions(additional_platform_directory=['/tmp/foo']))
 471         self.assertEqual(port.baseline_search_path()[0], '/tmp/foo')