Tools/Scripts/webkitpy/layout_tests/port/test.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the Google name nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 """Dummy Port implementation used for testing."""
  31 from __future__ import with_statement
  32
  33 import base64
  34 import time
  35
  36 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
  37 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  38 from webkitpy.common.host_mock import MockHost
  39 from webkitpy.common.system.filesystem_mock import MockFileSystem
  40
  41
  42 # This sets basic expectations for a test. Each individual expectation
  43 # can be overridden by a keyword argument in TestList.add().
  44 class TestInstance(object):
  45     def __init__(self, name):
  46         self.name = name
  47         self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
  48         self.crash = False
  49         self.web_process_crash = False
  50         self.exception = False
  51         self.hang = False
  52         self.keyboard = False
  53         self.error = ''
  54         self.timeout = False
  55         self.is_reftest = False
  56
  57         # The values of each field are treated as raw byte strings. They
  58         # will be converted to unicode strings where appropriate using
  59         # MockFileSystem.read_text_file().
  60         self.actual_text = self.base + '-txt'
  61         self.actual_checksum = self.base + '-checksum'
  62
  63         # We add the '\x8a' for the image file to prevent the value from
  64         # being treated as UTF-8 (the character is invalid)
  65         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
  66
  67         self.expected_text = self.actual_text
  68         self.expected_checksum = self.actual_checksum
  69         self.expected_image = self.actual_image
  70
  71         self.actual_audio = None
  72         self.expected_audio = None
  73
  74
  75 # This is an in-memory list of tests, what we want them to produce, and
  76 # what we want to claim are the expected results.
  77 class TestList(object):
  78     def __init__(self):
  79         self.tests = {}
  80
  81     def add(self, name, **kwargs):
  82         test = TestInstance(name)
  83         for key, value in kwargs.items():
  84             test.__dict__[key] = value
  85         self.tests[name] = test
  86
  87     def add_reftest(self, name, reference_name, same_image):
  88         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
  89         if same_image:
  90             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
  91         else:
  92             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
  93
  94     def keys(self):
  95         return self.tests.keys()
  96
  97     def __contains__(self, item):
  98         return item in self.tests
  99
 100     def __getitem__(self, item):
 101         return self.tests[item]
 102
 103
 104 def unit_test_list():
 105     tests = TestList()
 106     tests.add('failures/expected/checksum.html',
 107               actual_checksum='checksum_fail-checksum')
 108     tests.add('failures/expected/crash.html', crash=True)
 109     tests.add('failures/expected/exception.html', exception=True)
 110     tests.add('failures/expected/timeout.html', timeout=True)
 111     tests.add('failures/expected/hang.html', hang=True)
 112     tests.add('failures/expected/missing_text.html', expected_text=None)
 113     tests.add('failures/expected/image.html',
 114               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
 115               expected_image='image-pngtEXtchecksum\x00checksum-png')
 116     tests.add('failures/expected/image_checksum.html',
 117               actual_checksum='image_checksum_fail-checksum',
 118               actual_image='image_checksum_fail-png')
 119     tests.add('failures/expected/audio.html',
 120               actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
 121               actual_text=None, expected_text=None,
 122               actual_image=None, expected_image=None,
 123               actual_checksum=None, expected_checksum=None)
 124     tests.add('failures/expected/keyboard.html', keyboard=True)
 125     tests.add('failures/expected/missing_check.html',
 126               expected_checksum=None,
 127               expected_image=None)
 128     tests.add('failures/expected/missing_image.html', expected_image=None)
 129     tests.add('failures/expected/missing_audio.html', expected_audio=None,
 130               actual_text=None, expected_text=None,
 131               actual_image=None, expected_image=None,
 132               actual_checksum=None, expected_checksum=None)
 133     tests.add('failures/expected/missing_text.html', expected_text=None)
 134     tests.add('failures/expected/newlines_leading.html',
 135               expected_text="\nfoo\n", actual_text="foo\n")
 136     tests.add('failures/expected/newlines_trailing.html',
 137               expected_text="foo\n\n", actual_text="foo\n")
 138     tests.add('failures/expected/newlines_with_excess_CR.html',
 139               expected_text="foo\r\r\r\n", actual_text="foo\n")
 140     tests.add('failures/expected/text.html', actual_text='text_fail-png')
 141     tests.add('failures/unexpected/missing_text.html', expected_text=None)
 142     tests.add('failures/unexpected/missing_image.html', expected_image=None)
 143     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
 144   RenderView at (0,0) size 800x600
 145 layer at (0,0) size 800x34
 146   RenderBlock {HTML} at (0,0) size 800x34
 147     RenderBody {BODY} at (8,8) size 784x18
 148       RenderText {#text} at (0,0) size 133x18
 149         text run at (0,0) width 133: "This is an image test!"
 150 """, expected_text=None)
 151     tests.add('failures/unexpected/crash.html', crash=True)
 152     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
 153               error="mock-std-error-output")
 154     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
 155               error="mock-std-error-output")
 156     tests.add('failures/unexpected/text-image-checksum.html',
 157               actual_text='text-image-checksum_fail-txt',
 158               actual_checksum='text-image-checksum_fail-checksum')
 159     tests.add('failures/unexpected/checksum-with-matching-image.html',
 160               actual_checksum='text-image-checksum_fail-checksum')
 161     tests.add('failures/unexpected/timeout.html', timeout=True)
 162     tests.add('http/tests/passes/text.html')
 163     tests.add('http/tests/passes/image.html')
 164     tests.add('http/tests/ssl/text.html')
 165     tests.add('passes/error.html', error='stuff going to stderr')
 166     tests.add('passes/image.html')
 167     tests.add('passes/audio.html',
 168               actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
 169               actual_text=None, expected_text=None,
 170               actual_image=None, expected_image=None,
 171               actual_checksum=None, expected_checksum=None)
 172     tests.add('passes/platform_image.html')
 173     tests.add('passes/checksum_in_image.html',
 174               expected_checksum=None,
 175               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
 176
 177     # Text output files contain "\r\n" on Windows.  This may be
 178     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
 179     tests.add('passes/text.html',
 180               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
 181
 182     # For reftests.
 183     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
 184     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
 185     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
 186     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
 187     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
 188     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
 189     # FIXME: Add a reftest which crashes.
 190
 191     tests.add('websocket/tests/passes/text.html')
 192
 193     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
 194     tests.add('platform/test-snow-leopard/http/test.html')
 195     tests.add('platform/test-snow-leopard/websocket/test.html')
 196
 197     return tests
 198
 199
 200 # Here we use a non-standard location for the layout tests, to ensure that
 201 # this works. The path contains a '.' in the name because we've seen bugs
 202 # related to this before.
 203
 204 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
 205
 206
 207 # Here we synthesize an in-memory filesystem from the test list
 208 # in order to fully control the test output and to demonstrate that
 209 # we don't need a real filesystem to run the tests.
 210
 211 def unit_test_filesystem(files=None):
 212     """Return the FileSystem object used by the unit tests."""
 213     test_list = unit_test_list()
 214     files = files or {}
 215
 216     def add_file(files, test, suffix, contents):
 217         dirname = test.name[0:test.name.rfind('/')]
 218         base = test.base
 219         path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
 220         files[path] = contents
 221
 222     # Add each test and the expected output, if any.
 223     for test in test_list.tests.values():
 224         add_file(files, test, '.html', '')
 225         if test.is_reftest:
 226             continue
 227         if test.actual_audio:
 228             add_file(files, test, '-expected.wav', test.expected_audio)
 229             continue
 230
 231         add_file(files, test, '-expected.txt', test.expected_text)
 232         add_file(files, test, '-expected.png', test.expected_image)
 233
 234
 235     # Add the test_expectations file.
 236     files[LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'] = """
 237 WONTFIX : failures/expected/checksum.html = IMAGE
 238 WONTFIX : failures/expected/crash.html = CRASH
 239 WONTFIX : failures/expected/image.html = IMAGE
 240 WONTFIX : failures/expected/audio.html = AUDIO
 241 WONTFIX : failures/expected/image_checksum.html = IMAGE
 242 WONTFIX : failures/expected/mismatch.html = IMAGE
 243 WONTFIX : failures/expected/missing_check.html = MISSING PASS
 244 WONTFIX : failures/expected/missing_image.html = MISSING PASS
 245 WONTFIX : failures/expected/missing_audio.html = MISSING PASS
 246 WONTFIX : failures/expected/missing_text.html = MISSING PASS
 247 WONTFIX : failures/expected/newlines_leading.html = TEXT
 248 WONTFIX : failures/expected/newlines_trailing.html = TEXT
 249 WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
 250 WONTFIX : failures/expected/reftest.html = IMAGE
 251 WONTFIX : failures/expected/text.html = TEXT
 252 WONTFIX : failures/expected/timeout.html = TIMEOUT
 253 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
 254 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
 255 WONTFIX SKIP : failures/expected/exception.html = CRASH
 256 """
 257
 258     # FIXME: This test was only being ignored because of missing a leading '/'.
 259     # Fixing the typo causes several tests to assert, so disabling the test entirely.
 260     # Add in a file should be ignored by test_files.find().
 261     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
 262
 263     fs = MockFileSystem(files, dirs=set(['/mock-checkout']))  # Make sure at least the checkout_root exists as a directory.
 264     fs._tests = test_list
 265     return fs
 266
 267
 268 class TestPort(Port):
 269     """Test implementation of the Port interface."""
 270     ALL_BASELINE_VARIANTS = (
 271         'test-mac-snowleopard', 'test-mac-leopard',
 272         'test-win-win7', 'test-win-vista', 'test-win-xp',
 273         'test-linux-x86_64',
 274     )
 275
 276     def _set_default_overriding_none(self, dictionary, key, default):
 277         # dict.setdefault almost works, but won't actually override None values, which we want.
 278         if not dictionary.get(key):
 279             dictionary[key] = default
 280         return dictionary[key]
 281
 282     def __init__(self, host=None, port_name=None, **kwargs):
 283         if not port_name or port_name == 'test':
 284             port_name = 'test-mac-leopard'
 285
 286         host = host or MockHost()
 287         filesystem = self._set_default_overriding_none(kwargs, 'filesystem', unit_test_filesystem())
 288
 289         Port.__init__(self, host, port_name=port_name, **kwargs)
 290         self._results_directory = None
 291
 292         assert filesystem._tests
 293         self._tests = filesystem._tests
 294
 295         self._operating_system = 'mac'
 296         if port_name.startswith('test-win'):
 297             self._operating_system = 'win'
 298         elif port_name.startswith('test-linux'):
 299             self._operating_system = 'linux'
 300
 301         version_map = {
 302             'test-win-xp': 'xp',
 303             'test-win-win7': 'win7',
 304             'test-win-vista': 'vista',
 305             'test-mac-leopard': 'leopard',
 306             'test-mac-snowleopard': 'snowleopard',
 307             'test-linux-x86_64': 'lucid',
 308         }
 309         self._version = version_map[port_name]
 310
 311         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
 312
 313     def _path_to_driver(self):
 314         # This routine shouldn't normally be called, but it is called by
 315         # the mock_drt Driver. We return something, but make sure it's useless.
 316         return 'junk'
 317
 318     def baseline_search_path(self):
 319         search_paths = {
 320             'test-mac-snowleopard': ['test-mac-snowleopard'],
 321             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
 322             'test-win-win7': ['test-win-win7'],
 323             'test-win-vista': ['test-win-vista', 'test-win-win7'],
 324             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
 325             'test-linux-x86_64': ['test-linux', 'test-win-win7'],
 326         }
 327         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
 328
 329     def default_child_processes(self):
 330         return 1
 331
 332     def default_worker_model(self):
 333         return 'inline'
 334
 335     def check_build(self, needs_http):
 336         return True
 337
 338     def check_sys_deps(self, needs_http):
 339         return True
 340
 341     def default_configuration(self):
 342         return 'Release'
 343
 344     def diff_image(self, expected_contents, actual_contents):
 345         diffed = actual_contents != expected_contents
 346         if diffed:
 347             return ["< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1]
 348         return (None, 0)
 349
 350     def layout_tests_dir(self):
 351         return LAYOUT_TEST_DIR
 352
 353     def name(self):
 354         return self._name
 355
 356     def _path_to_wdiff(self):
 357         return None
 358
 359     def default_results_directory(self):
 360         return '/tmp/layout-test-results'
 361
 362     def setup_test_run(self):
 363         pass
 364
 365     def _driver_class(self):
 366         return TestDriver
 367
 368     def start_http_server(self):
 369         pass
 370
 371     def start_websocket_server(self):
 372         pass
 373
 374     def acquire_http_lock(self):
 375         pass
 376
 377     def stop_http_server(self):
 378         pass
 379
 380     def stop_websocket_server(self):
 381         pass
 382
 383     def release_http_lock(self):
 384         pass
 385
 386     def path_to_test_expectations_file(self):
 387         return self._expectations_path
 388
 389     def all_test_configurations(self):
 390         """Returns a sequence of the TestConfigurations the port supports."""
 391         # By default, we assume we want to test every graphics type in
 392         # every configuration on every system.
 393         test_configurations = []
 394         for version, architecture in self._all_systems():
 395             for build_type in self._all_build_types():
 396                 for graphics_type in self._all_graphics_types():
 397                     test_configurations.append(TestConfiguration(
 398                         version=version,
 399                         architecture=architecture,
 400                         build_type=build_type,
 401                         graphics_type=graphics_type))
 402         return test_configurations
 403
 404     def _all_systems(self):
 405         return (('leopard', 'x86'),
 406                 ('snowleopard', 'x86'),
 407                 ('xp', 'x86'),
 408                 ('vista', 'x86'),
 409                 ('win7', 'x86'),
 410                 ('lucid', 'x86'),
 411                 ('lucid', 'x86_64'))
 412
 413     def _all_build_types(self):
 414         return ('debug', 'release')
 415
 416     def _all_graphics_types(self):
 417         return ('cpu', 'gpu')
 418
 419     def configuration_specifier_macros(self):
 420         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
 421         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
 422
 423     def all_baseline_variants(self):
 424         return self.ALL_BASELINE_VARIANTS
 425
 426     # FIXME: These next two routines are copied from base.py with
 427     # the calls to path.abspath_to_uri() removed. We shouldn't have
 428     # to do this.
 429     def test_to_uri(self, test_name):
 430         """Convert a test file (which is an absolute path) to a URI."""
 431         LAYOUTTEST_HTTP_DIR = "http/tests/"
 432         LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
 433
 434         port = None
 435         use_ssl = False
 436
 437         relative_path = test_name
 438         if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
 439             or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
 440             relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
 441             port = 8000
 442
 443         # Make http/tests/local run as local files. This is to mimic the
 444         # logic in run-webkit-tests.
 445         #
 446         # TODO(dpranke): remove the media reference and the SSL reference?
 447         if (port and not relative_path.startswith("local/") and
 448             not relative_path.startswith("media/")):
 449             if relative_path.startswith("ssl/"):
 450                 port += 443
 451                 protocol = "https"
 452             else:
 453                 protocol = "http"
 454             return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)
 455
 456         return "file://" + self.abspath_for_test(test_name)
 457
 458     def abspath_for_test(self, test_name):
 459         return self.layout_tests_dir() + self.TEST_PATH_SEPARATOR + test_name
 460
 461     def uri_to_test_name(self, uri):
 462         """Return the base layout test name for a given URI.
 463
 464         This returns the test name for a given URI, e.g., if you passed in
 465         "file:///src/LayoutTests/fast/html/keygen.html" it would return
 466         "fast/html/keygen.html".
 467
 468         """
 469         test = uri
 470         if uri.startswith("file:///"):
 471             prefix = "file://" + self.layout_tests_dir() + "/"
 472             return test[len(prefix):]
 473
 474         if uri.startswith("http://127.0.0.1:8880/"):
 475             # websocket tests
 476             return test.replace('http://127.0.0.1:8880/', '')
 477
 478         if uri.startswith("http://"):
 479             # regular HTTP test
 480             return test.replace('http://127.0.0.1:8000/', 'http/tests/')
 481
 482         if uri.startswith("https://"):
 483             return test.replace('https://127.0.0.1:8443/', 'http/tests/')
 484
 485         raise NotImplementedError('unknown url type: %s' % uri)
 486
 487
 488 class TestDriver(Driver):
 489     """Test/Dummy implementation of the DumpRenderTree interface."""
 490
 491     def cmd_line(self):
 492         return [self._port._path_to_driver()] + self._port.get_option('additional_drt_flag', [])
 493
 494     def run_test(self, test_input):
 495         start_time = time.time()
 496         test_name = test_input.test_name
 497         test = self._port._tests[test_name]
 498         if test.keyboard:
 499             raise KeyboardInterrupt
 500         if test.exception:
 501             raise ValueError('exception from ' + test_name)
 502         if test.hang:
 503             time.sleep((float(test_input.timeout) * 4) / 1000.0)
 504
 505         audio = None
 506         if test.actual_audio:
 507             audio = base64.b64decode(test.actual_audio)
 508         crashed_process_name = None
 509         if test.crash:
 510             crashed_process_name = self._port.driver_name()
 511         elif test.web_process_crash:
 512             crashed_process_name = 'WebProcess'
 513         return DriverOutput(test.actual_text, test.actual_image,
 514             test.actual_checksum, audio, crash=test.crash or test.web_process_crash,
 515             crashed_process_name=crashed_process_name,
 516             test_time=time.time() - start_time, timeout=test.timeout, error=test.error)
 517
 518     def stop(self):
 519         pass