Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import optparse
  30 import sys
  31 import tempfile
  32 import unittest
  33
  34 from webkitpy.common.system.executive import Executive, ScriptError
  35 from webkitpy.common.system import executive_mock
  36 from webkitpy.common.system.filesystem_mock import MockFileSystem
  37 from webkitpy.common.system import outputcapture
  38 from webkitpy.common.system.path import abspath_to_uri
  39 from webkitpy.thirdparty.mock import Mock
  40 from webkitpy.tool.mocktool import MockOptions
  41 from webkitpy.common.system.executive_mock import MockExecutive
  42 from webkitpy.common.host_mock import MockHost
  43
  44 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
  45
  46 import config
  47 import config_mock
  48
  49
  50 class PortTest(unittest.TestCase):
  51     def make_port(self, host=None, **kwargs):
  52         host = host or MockHost()
  53         return Port(host, **kwargs)
  54
  55     def test_format_wdiff_output_as_html(self):
  56         output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
  57         html = self.make_port()._format_wdiff_output_as_html(output)
  58         expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
  59         self.assertEqual(html, expected_html)
  60
  61     def test_wdiff_command(self):
  62         port = self.make_port()
  63         port._path_to_wdiff = lambda: "/path/to/wdiff"
  64         command = port._wdiff_command("/actual/path", "/expected/path")
  65         expected_command = [
  66             "/path/to/wdiff",
  67             "--start-delete=##WDIFF_DEL##",
  68             "--end-delete=##WDIFF_END##",
  69             "--start-insert=##WDIFF_ADD##",
  70             "--end-insert=##WDIFF_END##",
  71             "/actual/path",
  72             "/expected/path",
  73         ]
  74         self.assertEqual(command, expected_command)
  75
  76     def _file_with_contents(self, contents, encoding="utf-8"):
  77         new_file = tempfile.NamedTemporaryFile()
  78         new_file.write(contents.encode(encoding))
  79         new_file.flush()
  80         return new_file
  81
  82     def test_pretty_patch_os_error(self):
  83         port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
  84         oc = outputcapture.OutputCapture()
  85         oc.capture_output()
  86         self.assertEqual(port.pretty_patch_text("patch.txt"),
  87                          port._pretty_patch_error_html)
  88
  89         # This tests repeated calls to make sure we cache the result.
  90         self.assertEqual(port.pretty_patch_text("patch.txt"),
  91                          port._pretty_patch_error_html)
  92         oc.restore_output()
  93
  94     def test_pretty_patch_script_error(self):
  95         # FIXME: This is some ugly white-box test hacking ...
  96         port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
  97         port._pretty_patch_available = True
  98         self.assertEqual(port.pretty_patch_text("patch.txt"),
  99                          port._pretty_patch_error_html)
 100
 101         # This tests repeated calls to make sure we cache the result.
 102         self.assertEqual(port.pretty_patch_text("patch.txt"),
 103                          port._pretty_patch_error_html)
 104
 105     def integration_test_run_wdiff(self):
 106         executive = Executive()
 107         # This may fail on some systems.  We could ask the port
 108         # object for the wdiff path, but since we don't know what
 109         # port object to use, this is sufficient for now.
 110         try:
 111             wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
 112         except Exception, e:
 113             wdiff_path = None
 114
 115         port = self.make_port(executive=executive)
 116         port._path_to_wdiff = lambda: wdiff_path
 117
 118         if wdiff_path:
 119             # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
 120             actual = self._file_with_contents(u"foo")
 121             expected = self._file_with_contents(u"bar")
 122             wdiff = port._run_wdiff(actual.name, expected.name)
 123             expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
 124             self.assertEqual(wdiff, expected_wdiff)
 125             # Running the full wdiff_text method should give the same result.
 126             port._wdiff_available = True  # In case it's somehow already disabled.
 127             wdiff = port.wdiff_text(actual.name, expected.name)
 128             self.assertEqual(wdiff, expected_wdiff)
 129             # wdiff should still be available after running wdiff_text with a valid diff.
 130             self.assertTrue(port._wdiff_available)
 131             actual.close()
 132             expected.close()
 133
 134             # Bogus paths should raise a script error.
 135             self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
 136             self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
 137             # wdiff will still be available after running wdiff_text with invalid paths.
 138             self.assertTrue(port._wdiff_available)
 139
 140         # If wdiff does not exist _run_wdiff should throw an OSError.
 141         port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
 142         self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
 143
 144         # wdiff_text should not throw an error if wdiff does not exist.
 145         self.assertEqual(port.wdiff_text("foo", "bar"), "")
 146         # However wdiff should not be available after running wdiff_text if wdiff is missing.
 147         self.assertFalse(port._wdiff_available)
 148
 149     def test_diff_text(self):
 150         port = self.make_port()
 151         # Make sure that we don't run into decoding exceptions when the
 152         # filenames are unicode, with regular or malformed input (expected or
 153         # actual input is always raw bytes, not unicode).
 154         port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
 155         port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
 156         port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
 157
 158         port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
 159         port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
 160
 161         # Though expected and actual files should always be read in with no
 162         # encoding (and be stored as str objects), test unicode inputs just to
 163         # be safe.
 164         port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
 165         port.diff_text(
 166             u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
 167
 168         # And make sure we actually get diff output.
 169         diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
 170         self.assertTrue('foo' in diff)
 171         self.assertTrue('bar' in diff)
 172         self.assertTrue('exp.txt' in diff)
 173         self.assertTrue('act.txt' in diff)
 174         self.assertFalse('nosuchthing' in diff)
 175
 176     def test_default_configuration_notfound(self):
 177         # Test that we delegate to the config object properly.
 178         port = self.make_port(config=config_mock.MockConfig(default_configuration='default'))
 179         self.assertEqual(port.default_configuration(), 'default')
 180
 181     def test_layout_tests_skipping(self):
 182         filesystem = MockFileSystem({
 183             '/mock-checkout/LayoutTests/media/video-zoom.html': '',
 184             '/mock-checkout/LayoutTests/foo/bar.html': '',
 185         })
 186         port = self.make_port(filesystem=filesystem)
 187         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
 188         self.assertTrue(port.skips_layout_test('foo/bar.html'))
 189         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
 190         self.assertFalse(port.skips_layout_test('foo/foo.html'))
 191
 192     def test_setup_test_run(self):
 193         port = self.make_port()
 194         # This routine is a no-op. We just test it for coverage.
 195         port.setup_test_run()
 196
 197     def test_test_dirs(self):
 198         filesystem = MockFileSystem({
 199             '/mock-checkout/LayoutTests/canvas/test': '',
 200             '/mock-checkout/LayoutTests/css2.1/test': '',
 201         })
 202         port = self.make_port(filesystem=filesystem)
 203         dirs = port.test_dirs()
 204         self.assertTrue('canvas' in dirs)
 205         self.assertTrue('css2.1' in dirs)
 206
 207     def test_test_to_uri(self):
 208         port = self.make_port()
 209         layout_test_dir = port.layout_tests_dir()
 210         test = 'foo/bar.html'
 211         path = port._filesystem.join(layout_test_dir, test)
 212         if sys.platform == 'win32':
 213             path = path.replace("\\", "/")
 214
 215         self.assertEqual(port.test_to_uri(test), abspath_to_uri(path))
 216
 217     def test_get_option__set(self):
 218         options, args = optparse.OptionParser().parse_args([])
 219         options.foo = 'bar'
 220         port = self.make_port(options=options)
 221         self.assertEqual(port.get_option('foo'), 'bar')
 222
 223     def test_get_option__unset(self):
 224         port = self.make_port()
 225         self.assertEqual(port.get_option('foo'), None)
 226
 227     def test_get_option__default(self):
 228         port = self.make_port()
 229         self.assertEqual(port.get_option('foo', 'bar'), 'bar')
 230
 231     def test_name__unset(self):
 232         port = self.make_port()
 233         self.assertEqual(port.name(), None)
 234
 235     def test_name__set(self):
 236         port = self.make_port(port_name='foo')
 237         self.assertEqual(port.name(), 'foo')
 238
 239     def test_additional_platform_directory(self):
 240         filesystem = MockFileSystem()
 241         port = self.make_port(port_name='foo', filesystem=filesystem)
 242         port.baseline_search_path = lambda: ['LayoutTests/platform/foo']
 243         layout_test_dir = port.layout_tests_dir()
 244         test_file = 'fast/test.html'
 245
 246         # No additional platform directory
 247         self.assertEqual(
 248             port.expected_baselines(test_file, '.txt'),
 249             [(None, 'fast/test-expected.txt')])
 250         self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
 251
 252         # Simple additional platform directory
 253         port._options.additional_platform_directory = ['/tmp/local-baselines']
 254         filesystem.files = {
 255             '/tmp/local-baselines/fast/test-expected.txt': 'foo',
 256         }
 257         self.assertEqual(
 258             port.expected_baselines(test_file, '.txt'),
 259             [('/tmp/local-baselines', 'fast/test-expected.txt')])
 260         self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
 261
 262         # Multiple additional platform directories
 263         port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
 264         self.assertEqual(
 265             port.expected_baselines(test_file, '.txt'),
 266             [('/tmp/local-baselines', 'fast/test-expected.txt')])
 267         self.assertEqual(port.baseline_path(), '/foo')
 268
 269     def test_uses_test_expectations_file(self):
 270         filesystem = MockFileSystem()
 271         port = self.make_port(port_name='foo', filesystem=filesystem)
 272         port.path_to_test_expectations_file = lambda: '/mock-results/test_expectations.txt'
 273         self.assertFalse(port.uses_test_expectations_file())
 274         port._filesystem = MockFileSystem({'/mock-results/test_expectations.txt': ''})
 275         self.assertTrue(port.uses_test_expectations_file())
 276
 277
 278 class VirtualTest(unittest.TestCase):
 279     """Tests that various methods expected to be virtual are."""
 280     def assertVirtual(self, method, *args, **kwargs):
 281         self.assertRaises(NotImplementedError, method, *args, **kwargs)
 282
 283     def test_virtual_methods(self):
 284         port = Port(MockHost())
 285         self.assertVirtual(port.baseline_path)
 286         self.assertVirtual(port.baseline_search_path)
 287         self.assertVirtual(port.check_build, None)
 288         self.assertVirtual(port.check_image_diff)
 289         self.assertVirtual(port.create_driver, 0)
 290         self.assertVirtual(port.diff_image, None, None)
 291         self.assertVirtual(port.path_to_test_expectations_file)
 292         self.assertVirtual(port.default_results_directory)
 293         self.assertVirtual(port.test_expectations)
 294         self.assertVirtual(port._path_to_apache)
 295         self.assertVirtual(port._path_to_apache_config_file)
 296         self.assertVirtual(port._path_to_driver)
 297         self.assertVirtual(port._path_to_helper)
 298         self.assertVirtual(port._path_to_image_diff)
 299         self.assertVirtual(port._path_to_lighttpd)
 300         self.assertVirtual(port._path_to_lighttpd_modules)
 301         self.assertVirtual(port._path_to_lighttpd_php)
 302         self.assertVirtual(port._path_to_wdiff)
 303
 304     def test_virtual_driver_methods(self):
 305         driver = Driver(Port(MockHost()), None, pixel_tests=False)
 306         self.assertVirtual(driver.run_test, None)
 307         self.assertVirtual(driver.stop)
 308         self.assertVirtual(driver.cmd_line)
 309
 310
 311 # FIXME: This should be moved to driver_unittest.py or just deleted.
 312 class DriverTest(unittest.TestCase):
 313
 314     def _assert_wrapper(self, wrapper_string, expected_wrapper):
 315         wrapper = Driver(Port(MockHost()), None, pixel_tests=False)._command_wrapper(wrapper_string)
 316         self.assertEqual(wrapper, expected_wrapper)
 317
 318     def test_command_wrapper(self):
 319         self._assert_wrapper(None, [])
 320         self._assert_wrapper("valgrind", ["valgrind"])
 321
 322         # Validate that shlex works as expected.
 323         command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
 324         expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
 325         self._assert_wrapper(command_with_spaces, expected_parse)
 326
 327
 328 if __name__ == '__main__':
 329     unittest.main()