Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 # Copyright (C) 2011 Apple Inc. All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions are
   8 # met:
   9 #
  10 #     * Redistributions of source code must retain the above copyright
  11 # notice, this list of conditions and the following disclaimer.
  12 #     * Redistributions in binary form must reproduce the above
  13 # copyright notice, this list of conditions and the following disclaimer
  14 # in the documentation and/or other materials provided with the
  15 # distribution.
  16 #     * Neither the name of Google Inc. nor the names of its
  17 # contributors may be used to endorse or promote products derived from
  18 # this software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31
  32 """Unit tests for run_webkit_tests."""
  33
  34 from __future__ import with_statement
  35
  36 import codecs
  37 import itertools
  38 import logging
  39 import os
  40 import Queue
  41 import sys
  42 import thread
  43 import time
  44 import threading
  45 import unittest
  46
  47 try:
  48     import multiprocessing
  49 except ImportError:
  50     multiprocessing = None
  51
  52 try:
  53     import json
  54 except ImportError:
  55     # python 2.5 compatibility
  56     import webkitpy.thirdparty.simplejson as json
  57
  58 # FIXME: remove this when we fix test-webkitpy to work properly on cygwin
  59 # (bug 63846).
  60 SHOULD_TEST_PROCESSES = multiprocessing and sys.platform not in ('cygwin', 'win32')
  61
  62 from webkitpy.common import array_stream
  63 from webkitpy.common.system import outputcapture
  64 from webkitpy.common.host_mock import MockHost
  65
  66 from webkitpy.layout_tests import port
  67 from webkitpy.layout_tests import run_webkit_tests
  68 from webkitpy.layout_tests.port.test import TestPort, TestDriver, unit_test_filesystem
  69 from webkitpy.layout_tests.port.test_files import is_reference_html_file
  70 from webkitpy.python24.versioning import compare_version
  71 from webkitpy.test.skip import skip_if
  72
  73
  74 def parse_args(extra_args=None, record_results=False, tests_included=False, new_results=False, print_nothing=True):
  75     extra_args = extra_args or []
  76     if print_nothing:
  77         args = ['--print', 'nothing']
  78     else:
  79         args = []
  80     if not '--platform' in extra_args:
  81         args.extend(['--platform', 'test'])
  82     if not record_results:
  83         args.append('--no-record-results')
  84     if not new_results:
  85         args.append('--no-new-test-results')
  86
  87     if not '--child-processes' in extra_args and not '--worker-model' in extra_args:
  88         args.extend(['--worker-model', 'inline'])
  89     args.extend(extra_args)
  90     if not tests_included:
  91         # We use the glob to test that globbing works.
  92         args.extend(['passes',
  93                      'http/tests',
  94                      'websocket/tests',
  95                      'failures/expected/*'])
  96     return run_webkit_tests.parse_args(args)
  97
  98
  99 def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None):
 100     options, parsed_args = parse_args(extra_args, record_results, tests_included)
 101     filesystem = filesystem or unit_test_filesystem()
 102     if not port_obj:
 103         host = MockHost()
 104         port_obj = host.port_factory.get(port_name=options.platform, options=options, filesystem=filesystem)
 105     buildbot_output = array_stream.ArrayStream()
 106     regular_output = array_stream.ArrayStream()
 107     res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output)
 108     return res == 0 and regular_output.empty() and buildbot_output.empty()
 109
 110
 111 def logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None, new_results=False):
 112     options, parsed_args = parse_args(extra_args=extra_args,
 113                                       record_results=record_results,
 114                                       tests_included=tests_included,
 115                                       print_nothing=False, new_results=new_results)
 116     host = MockHost()
 117     filesystem = filesystem or unit_test_filesystem()
 118     if not port_obj:
 119         port_obj = host.port_factory.get(port_name=options.platform, options=options, filesystem=filesystem)
 120
 121     res, buildbot_output, regular_output = run_and_capture(port_obj, options, parsed_args)
 122     return (res, buildbot_output, regular_output, host.user)
 123
 124
 125 def run_and_capture(port_obj, options, parsed_args):
 126     oc = outputcapture.OutputCapture()
 127     try:
 128         oc.capture_output()
 129         buildbot_output = array_stream.ArrayStream()
 130         regular_output = array_stream.ArrayStream()
 131         res = run_webkit_tests.run(port_obj, options, parsed_args,
 132                                    buildbot_output=buildbot_output,
 133                                    regular_output=regular_output)
 134     finally:
 135         oc.restore_output()
 136     return (res, buildbot_output, regular_output)
 137
 138
 139 def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
 140                   filesystem=None, include_reference_html=False):
 141     extra_args = extra_args or []
 142     if not tests_included:
 143         # Not including http tests since they get run out of order (that
 144         # behavior has its own test, see test_get_test_file_queue)
 145         extra_args = ['passes', 'failures'] + extra_args
 146     options, parsed_args = parse_args(extra_args, tests_included=True)
 147
 148     host = MockHost()
 149
 150     test_batches = []
 151
 152
 153     class RecordingTestDriver(TestDriver):
 154         def __init__(self, port, worker_number):
 155             TestDriver.__init__(self, port, worker_number, pixel_tests=port.get_option('pixel_test'))
 156             self._current_test_batch = None
 157
 158         def stop(self):
 159             self._current_test_batch = None
 160
 161         def run_test(self, test_input):
 162             if self._current_test_batch is None:
 163                 self._current_test_batch = []
 164                 test_batches.append(self._current_test_batch)
 165             test_name = test_input.test_name
 166             # In case of reftest, one test calls the driver's run_test() twice.
 167             # We should not add a reference html used by reftests to tests unless include_reference_html parameter
 168             # is explicitly given.
 169             if include_reference_html or not is_reference_html_file(test_input.test_name):
 170                 self._current_test_batch.append(test_name)
 171             return TestDriver.run_test(self, test_input)
 172
 173     class RecordingTestPort(TestPort):
 174         def create_driver(self, worker_number):
 175             return RecordingTestDriver(self, worker_number)
 176
 177     recording_port = RecordingTestPort(options=options, host=host, filesystem=filesystem)
 178     run_and_capture(recording_port, options, parsed_args)
 179
 180     if flatten_batches:
 181         return list(itertools.chain(*test_batches))
 182
 183     return test_batches
 184
 185
 186 class MainTest(unittest.TestCase):
 187     def test_accelerated_compositing(self):
 188         # This just tests that we recognize the command line args
 189         self.assertTrue(passing_run(['--accelerated-video']))
 190         self.assertTrue(passing_run(['--no-accelerated-video']))
 191
 192     def test_accelerated_2d_canvas(self):
 193         # This just tests that we recognize the command line args
 194         self.assertTrue(passing_run(['--accelerated-2d-canvas']))
 195         self.assertTrue(passing_run(['--no-accelerated-2d-canvas']))
 196
 197     def test_basic(self):
 198         self.assertTrue(passing_run())
 199
 200     def test_batch_size(self):
 201         batch_tests_run = get_tests_run(['--batch-size', '2'])
 202         for batch in batch_tests_run:
 203             self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
 204
 205     def test_child_process_1(self):
 206         if SHOULD_TEST_PROCESSES:
 207             _, _, regular_output, _ = logging_run(
 208                 ['--print', 'config', '--worker-model', 'processes', '--child-processes', '1'])
 209             self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()]))
 210
 211     def test_child_processes_2(self):
 212         # This test seems to fail on win32.
 213         if sys.platform == 'win32':
 214             return
 215         if SHOULD_TEST_PROCESSES:
 216             _, _, regular_output, _ = logging_run(
 217                 ['--print', 'config', '--worker-model', 'processes', '--child-processes', '2'])
 218             self.assertTrue(any(['Running 2 ' in line for line in regular_output.get()]))
 219
 220     def test_child_processes_min(self):
 221         if SHOULD_TEST_PROCESSES:
 222             _, _, regular_output, _ = logging_run(
 223                 ['--print', 'config', '--worker-model', 'processes', '--child-processes', '2', 'passes'],
 224                 tests_included=True)
 225             self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()]))
 226
 227     def test_dryrun(self):
 228         batch_tests_run = get_tests_run(['--dry-run'])
 229         self.assertEqual(batch_tests_run, [])
 230
 231         batch_tests_run = get_tests_run(['-n'])
 232         self.assertEqual(batch_tests_run, [])
 233
 234     def test_exception_raised(self):
 235         # Exceptions raised by a worker are treated differently depending on
 236         # whether they are in-process or out. inline exceptions work as normal,
 237         # which allows us to get the full stack trace and traceback from the
 238         # worker. The downside to this is that it could be any error, but this
 239         # is actually useful in testing, which is what --worker-model=inline is
 240         # usually used for.
 241         #
 242         # Exceptions raised in a separate process are re-packaged into
 243         # WorkerExceptions, which have a string capture of the stack which can
 244         # be printed, but don't display properly in the unit test exception handlers.
 245         self.assertRaises(ValueError, logging_run,
 246             ['failures/expected/exception.html'], tests_included=True)
 247
 248         if SHOULD_TEST_PROCESSES:
 249             self.assertRaises(run_webkit_tests.WorkerException, logging_run,
 250                 ['--worker-model', 'processes', 'failures/expected/exception.html'], tests_included=True)
 251
 252     def test_full_results_html(self):
 253         # FIXME: verify html?
 254         res, out, err, user = logging_run(['--full-results-html'])
 255         self.assertEqual(res, 0)
 256
 257     def test_help_printing(self):
 258         res, out, err, user = logging_run(['--help-printing'])
 259         self.assertEqual(res, 0)
 260         self.assertTrue(out.empty())
 261         self.assertFalse(err.empty())
 262
 263     def test_hung_thread(self):
 264         res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50',
 265                                           'failures/expected/hang.html'],
 266                                           tests_included=True)
 267         self.assertEqual(res, 0)
 268         self.assertFalse(out.empty())
 269         self.assertFalse(err.empty())
 270
 271     def test_keyboard_interrupt(self):
 272         # Note that this also tests running a test marked as SKIP if
 273         # you specify it explicitly.
 274         self.assertRaises(KeyboardInterrupt, logging_run,
 275             ['failures/expected/keyboard.html'], tests_included=True)
 276
 277     def test_keyboard_interrupt_inline_worker_model(self):
 278         self.assertRaises(KeyboardInterrupt, logging_run,
 279             ['failures/expected/keyboard.html', '--worker-model', 'inline'],
 280             tests_included=True)
 281
 282     def test_lint_test_files(self):
 283         res, out, err, user = logging_run(['--lint-test-files'])
 284         self.assertEqual(res, 0)
 285         self.assertTrue(out.empty())
 286         self.assertTrue(any(['Lint succeeded' in msg for msg in err.get()]))
 287
 288     def test_lint_test_files__errors(self):
 289         options, parsed_args = parse_args(['--lint-test-files'])
 290         host = MockHost()
 291         port_obj = host.port_factory.get(options.platform, options=options)
 292         port_obj.test_expectations = lambda: "# syntax error"
 293         res, out, err = run_and_capture(port_obj, options, parsed_args)
 294
 295         self.assertEqual(res, -1)
 296         self.assertTrue(out.empty())
 297         self.assertTrue(any(['Lint failed' in msg for msg in err.get()]))
 298
 299     def test_no_tests_found(self):
 300         res, out, err, user = logging_run(['resources'], tests_included=True)
 301         self.assertEqual(res, -1)
 302         self.assertTrue(out.empty())
 303         self.assertTrue('No tests to run.\n' in err.get())
 304
 305     def test_no_tests_found_2(self):
 306         res, out, err, user = logging_run(['foo'], tests_included=True)
 307         self.assertEqual(res, -1)
 308         self.assertTrue(out.empty())
 309         self.assertTrue('No tests to run.\n' in err.get())
 310
 311     def test_randomize_order(self):
 312         # FIXME: verify order was shuffled
 313         self.assertTrue(passing_run(['--randomize-order']))
 314
 315     def test_gc_between_tests(self):
 316         self.assertTrue(passing_run(['--gc-between-tests']))
 317
 318     def test_complex_text(self):
 319         self.assertTrue(passing_run(['--complex-text']))
 320
 321     def test_threaded(self):
 322         self.assertTrue(passing_run(['--threaded']))
 323
 324     def test_repeat_each(self):
 325         tests_to_run = ['passes/image.html', 'passes/text.html']
 326         tests_run = get_tests_run(['--repeat-each', '2'] + tests_to_run, tests_included=True, flatten_batches=True)
 327         self.assertEquals(tests_run, ['passes/image.html', 'passes/image.html', 'passes/text.html', 'passes/text.html'])
 328
 329     def test_iterations(self):
 330         tests_to_run = ['passes/image.html', 'passes/text.html']
 331         tests_run = get_tests_run(['--iterations', '2'] + tests_to_run, tests_included=True, flatten_batches=True)
 332         self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html', 'passes/image.html', 'passes/text.html'])
 333
 334     def test_run_chunk(self):
 335         # Test that we actually select the right chunk
 336         all_tests_run = get_tests_run(flatten_batches=True)
 337         chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True)
 338         self.assertEquals(all_tests_run[4:8], chunk_tests_run)
 339
 340         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
 341         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
 342         chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True)
 343         self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
 344
 345     def test_run_force(self):
 346         # This raises an exception because we run
 347         # failures/expected/exception.html, which is normally SKIPped.
 348
 349         # See also the comments in test_exception_raised() about ValueError vs. WorkerException.
 350         self.assertRaises(ValueError, logging_run, ['--force'])
 351
 352     def test_run_part(self):
 353         # Test that we actually select the right part
 354         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
 355         tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True)
 356         self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run)
 357
 358         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
 359         # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
 360         # last part repeats the first two tests).
 361         chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True)
 362         self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run)
 363
 364     def test_run_singly(self):
 365         batch_tests_run = get_tests_run(['--run-singly'])
 366         for batch in batch_tests_run:
 367             self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
 368
 369     def test_skip_failing_tests(self):
 370         batches = get_tests_run(['--skip-failing-tests'])
 371         has_passes_text = False
 372         for batch in batches:
 373             self.assertFalse('failures/expected/text.html' in batch)
 374             has_passes_text = has_passes_text or ('passes/text.html' in batch)
 375         self.assertTrue(has_passes_text)
 376
 377     def test_run_singly_actually_runs_tests(self):
 378         res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
 379         self.assertEquals(res, 7)
 380
 381     def test_single_file(self):
 382         # FIXME: We should consider replacing more of the get_tests_run()-style tests
 383         # with tests that read the tests_run* files, like this one.
 384         fs = unit_test_filesystem()
 385         tests_run = passing_run(['passes/text.html'], tests_included=True, filesystem=fs)
 386         self.assertEquals(fs.read_text_file('/tmp/layout-test-results/tests_run0.txt'),
 387                           'passes/text.html\n')
 388
 389     def test_single_file_with_prefix(self):
 390         tests_run = get_tests_run(['LayoutTests/passes/text.html'], tests_included=True, flatten_batches=True)
 391         self.assertEquals(['passes/text.html'], tests_run)
 392
 393     def test_single_skipped_file(self):
 394         tests_run = get_tests_run(['failures/expected/keybaord.html'], tests_included=True, flatten_batches=True)
 395         self.assertEquals([], tests_run)
 396
 397     def test_stderr_is_saved(self):
 398         fs = unit_test_filesystem()
 399         self.assertTrue(passing_run(filesystem=fs))
 400         self.assertEquals(fs.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
 401                           'stuff going to stderr')
 402
 403     def test_test_list(self):
 404         fs = unit_test_filesystem()
 405         filename = '/tmp/foo.txt'
 406         fs.write_text_file(filename, 'passes/text.html')
 407         tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs)
 408         self.assertEquals(['passes/text.html'], tests_run)
 409         fs.remove(filename)
 410         res, out, err, user = logging_run(['--test-list=%s' % filename],
 411                                           tests_included=True, filesystem=fs)
 412         self.assertEqual(res, -1)
 413         self.assertFalse(err.empty())
 414
 415     def test_test_list_with_prefix(self):
 416         fs = unit_test_filesystem()
 417         filename = '/tmp/foo.txt'
 418         fs.write_text_file(filename, 'LayoutTests/passes/text.html')
 419         tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs)
 420         self.assertEquals(['passes/text.html'], tests_run)
 421
 422     def test_unexpected_failures(self):
 423         # Run tests including the unexpected failures.
 424         self._url_opened = None
 425         res, out, err, user = logging_run(tests_included=True)
 426
 427         # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
 428         # FIXME: It's nice to have a routine in port/test.py that returns this number.
 429         unexpected_tests_count = 7
 430
 431         self.assertEqual(res, unexpected_tests_count)
 432         self.assertFalse(out.empty())
 433         self.assertFalse(err.empty())
 434         self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
 435
 436     def test_missing_and_unexpected_results(self):
 437         # Test that we update expectations in place. If the expectation
 438         # is missing, update the expected generic location.
 439         fs = unit_test_filesystem()
 440         res, out, err, _ = logging_run(['--no-show-results',
 441             'failures/expected/missing_image.html',
 442             'failures/unexpected/missing_text.html',
 443             'failures/unexpected/text-image-checksum.html'],
 444             tests_included=True, filesystem=fs, record_results=True)
 445         file_list = fs.written_files.keys()
 446         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
 447         self.assertEquals(res, 1)
 448         expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
 449         json_string = fs.read_text_file('/tmp/layout-test-results/full_results.json')
 450         self.assertTrue(json_string.find(expected_token) != -1)
 451         self.assertTrue(json_string.find('"num_regression":1') == -1)
 452         self.assertTrue(json_string.find('"num_flaky":1') == -1)
 453         self.assertTrue(json_string.find('"num_missing":1') != -1)
 454
 455     def test_missing_and_unexpected_results_with_custom_exit_code(self):
 456         # Test that we update expectations in place. If the expectation
 457         # is missing, update the expected generic location.
 458         fs = unit_test_filesystem()
 459
 460         class CustomExitCodePort(TestPort):
 461             def exit_code_from_summarized_results(self, unexpected_results):
 462                 return unexpected_results['num_regressions'] + unexpected_results['num_missing']
 463
 464         options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
 465         test_port = CustomExitCodePort(options=options)
 466         res, out, err, _ = logging_run(['--no-show-results',
 467             'failures/expected/missing_image.html',
 468             'failures/unexpected/missing_text.html',
 469             'failures/unexpected/text-image-checksum.html'],
 470             tests_included=True, filesystem=fs, record_results=True, port_obj=test_port)
 471         self.assertEquals(res, 2)
 472
 473     def test_crash_with_stderr(self):
 474         fs = unit_test_filesystem()
 475         res, buildbot_output, regular_output, user = logging_run([
 476                 'failures/unexpected/crash-with-stderr.html',
 477             ],
 478             tests_included=True,
 479             record_results=True,
 480             filesystem=fs)
 481         self.assertTrue(fs.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
 482
 483     def test_no_image_failure_with_image_diff(self):
 484         fs = unit_test_filesystem()
 485         res, buildbot_output, regular_output, user = logging_run([
 486                 'failures/unexpected/checksum-with-matching-image.html',
 487             ],
 488             tests_included=True,
 489             record_results=True,
 490             filesystem=fs)
 491         self.assertTrue(fs.read_text_file('/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1)
 492
 493     def test_crash_log(self):
 494         mock_crash_report = 'mock-crash-report'
 495         fs = unit_test_filesystem()
 496         fs.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash', mock_crash_report)
 497         res, buildbot_output, regular_output, user = logging_run([
 498                 'failures/unexpected/crash-with-stderr.html',
 499             ],
 500             tests_included=True,
 501             record_results=True,
 502             filesystem=fs)
 503         expected_crash_log = mock_crash_report
 504         # Currently CrashLog uploading only works on Darwin.
 505         if sys.platform != "darwin":
 506             expected_crash_log = "mock-std-error-output"
 507         self.assertEquals(fs.read_text_file('/tmp/layout-test-results/failures/unexpected/crash-with-stderr-crash-log.txt'), expected_crash_log)
 508
 509     def test_web_process_crash_log(self):
 510         mock_crash_report = 'mock-crash-report'
 511         fs = unit_test_filesystem()
 512         fs.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/WebProcess_2011-06-13-150719_quadzen.crash', mock_crash_report)
 513         res, buildbot_output, regular_output, user = logging_run([
 514                 'failures/unexpected/web-process-crash-with-stderr.html',
 515             ],
 516             tests_included=True,
 517             record_results=True,
 518             filesystem=fs)
 519         expected_crash_log = mock_crash_report
 520         # Currently CrashLog uploading only works on Darwin.
 521         if sys.platform != "darwin":
 522             expected_crash_log = "mock-std-error-output"
 523         self.assertEquals(fs.read_text_file('/tmp/layout-test-results/failures/unexpected/web-process-crash-with-stderr-crash-log.txt'), expected_crash_log)
 524
 525     def test_exit_after_n_failures_upload(self):
 526         fs = unit_test_filesystem()
 527         res, buildbot_output, regular_output, user = logging_run([
 528                 'failures/unexpected/text-image-checksum.html',
 529                 'passes/text.html',
 530                 '--exit-after-n-failures', '1',
 531             ],
 532             tests_included=True,
 533             record_results=True,
 534             filesystem=fs)
 535         self.assertTrue('/tmp/layout-test-results/incremental_results.json' in fs.files)
 536
 537     def test_exit_after_n_failures(self):
 538         # Unexpected failures should result in tests stopping.
 539         tests_run = get_tests_run([
 540                 'failures/unexpected/text-image-checksum.html',
 541                 'passes/text.html',
 542                 '--exit-after-n-failures', '1',
 543             ],
 544             tests_included=True,
 545             flatten_batches=True)
 546         self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run)
 547
 548         # But we'll keep going for expected ones.
 549         tests_run = get_tests_run([
 550                 'failures/expected/text.html',
 551                 'passes/text.html',
 552                 '--exit-after-n-failures', '1',
 553             ],
 554             tests_included=True,
 555             flatten_batches=True)
 556         self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run)
 557
 558     def test_exit_after_n_crashes(self):
 559         # Unexpected crashes should result in tests stopping.
 560         tests_run = get_tests_run([
 561                 'failures/unexpected/crash.html',
 562                 'passes/text.html',
 563                 '--exit-after-n-crashes-or-timeouts', '1',
 564             ],
 565             tests_included=True,
 566             flatten_batches=True)
 567         self.assertEquals(['failures/unexpected/crash.html'], tests_run)
 568
 569         # Same with timeouts.
 570         tests_run = get_tests_run([
 571                 'failures/unexpected/timeout.html',
 572                 'passes/text.html',
 573                 '--exit-after-n-crashes-or-timeouts', '1',
 574             ],
 575             tests_included=True,
 576             flatten_batches=True)
 577         self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
 578
 579         # But we'll keep going for expected ones.
 580         tests_run = get_tests_run([
 581                 'failures/expected/crash.html',
 582                 'passes/text.html',
 583                 '--exit-after-n-crashes-or-timeouts', '1',
 584             ],
 585             tests_included=True,
 586             flatten_batches=True)
 587         self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run)
 588
 589     def test_exit_after_n_crashes_inline_worker_model(self):
 590         tests_run = get_tests_run([
 591                 'failures/unexpected/timeout.html',
 592                 'passes/text.html',
 593                 '--exit-after-n-crashes-or-timeouts', '1',
 594                 '--worker-model', 'inline',
 595             ],
 596             tests_included=True,
 597             flatten_batches=True)
 598         self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
 599
 600     def test_results_directory_absolute(self):
 601         # We run a configuration that should fail, to generate output, then
 602         # look for what the output results url was.
 603
 604         fs = unit_test_filesystem()
 605         with fs.mkdtemp() as tmpdir:
 606             res, out, err, user = logging_run(['--results-directory=' + str(tmpdir)],
 607                                               tests_included=True, filesystem=fs)
 608             self.assertEqual(user.opened_urls, [fs.join(tmpdir, 'results.html')])
 609
 610     def test_results_directory_default(self):
 611         # We run a configuration that should fail, to generate output, then
 612         # look for what the output results url was.
 613
 614         # This is the default location.
 615         res, out, err, user = logging_run(tests_included=True)
 616         self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
 617
 618     def test_results_directory_relative(self):
 619         # We run a configuration that should fail, to generate output, then
 620         # look for what the output results url was.
 621         fs = unit_test_filesystem()
 622         fs.maybe_make_directory('/tmp/cwd')
 623         fs.chdir('/tmp/cwd')
 624         res, out, err, user = logging_run(['--results-directory=foo'],
 625                                           tests_included=True, filesystem=fs)
 626         self.assertEqual(user.opened_urls, ['/tmp/cwd/foo/results.html'])
 627
 628     # These next tests test that we run the tests in ascending alphabetical
 629     # order per directory. HTTP tests are sharded separately from other tests,
 630     # so we have to test both.
 631     def assert_run_order(self, worker_model, child_processes='1'):
 632         tests_run = get_tests_run(['--worker-model', worker_model,
 633             '--child-processes', child_processes, 'passes'],
 634             tests_included=True, flatten_batches=True)
 635         self.assertEquals(tests_run, sorted(tests_run))
 636
 637         tests_run = get_tests_run(['--worker-model', worker_model,
 638             '--child-processes', child_processes, 'http/tests/passes'],
 639             tests_included=True, flatten_batches=True)
 640         self.assertEquals(tests_run, sorted(tests_run))
 641
 642     def test_run_order__inline(self):
 643         self.assert_run_order('inline')
 644
 645     def test_tolerance(self):
 646         class ImageDiffTestPort(TestPort):
 647             def diff_image(self, expected_contents, actual_contents):
 648                 self.tolerance_used_for_diff_image = self._options.tolerance
 649                 return (True, 1)
 650
 651         def get_port_for_run(args):
 652             options, parsed_args = run_webkit_tests.parse_args(args)
 653             test_port = ImageDiffTestPort(options=options)
 654             res = passing_run(args, port_obj=test_port, tests_included=True)
 655             self.assertTrue(res)
 656             return test_port
 657
 658         base_args = ['--pixel-tests', '--no-new-test-results', 'failures/expected/*']
 659
 660         # If we pass in an explicit tolerance argument, then that will be used.
 661         test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
 662         self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
 663         test_port = get_port_for_run(base_args + ['--tolerance', '0'])
 664         self.assertEqual(0, test_port.tolerance_used_for_diff_image)
 665
 666         # Otherwise the port's default tolerance behavior (including ignoring it)
 667         # should be used.
 668         test_port = get_port_for_run(base_args)
 669         self.assertEqual(None, test_port.tolerance_used_for_diff_image)
 670
 671     def test_worker_model__inline(self):
 672         self.assertTrue(passing_run(['--worker-model', 'inline']))
 673
 674     def test_worker_model__inline_with_child_processes(self):
 675         res, out, err, user = logging_run(['--worker-model', 'inline',
 676                                            '--child-processes', '2'])
 677         self.assertEqual(res, 0)
 678         self.assertTrue('--worker-model=inline overrides --child-processes\n' in err.get())
 679
 680     def test_worker_model__processes(self):
 681         if SHOULD_TEST_PROCESSES:
 682             self.assertTrue(passing_run(['--worker-model', 'processes']))
 683
 684     def test_worker_model__processes_and_dry_run(self):
 685         if SHOULD_TEST_PROCESSES:
 686             self.assertTrue(passing_run(['--worker-model', 'processes', '--dry-run']))
 687
 688     def test_worker_model__unknown(self):
 689         self.assertRaises(ValueError, logging_run, ['--worker-model', 'unknown'])
 690
 691     def test_reftest_run(self):
 692         tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True)
 693         self.assertEquals(['passes/reftest.html'], tests_run)
 694
 695     def test_reftest_run_reftests_if_pixel_tests_are_disabled(self):
 696         tests_run = get_tests_run(['--no-pixel-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
 697         self.assertEquals(['passes/reftest.html'], tests_run)
 698
 699     def test_reftest_skip_reftests_if_no_ref_tests(self):
 700         tests_run = get_tests_run(['--no-ref-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
 701         self.assertEquals([], tests_run)
 702         tests_run = get_tests_run(['--no-ref-tests', '--no-pixel-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
 703         self.assertEquals([], tests_run)
 704
 705     def test_reftest_expected_html_should_be_ignored(self):
 706         tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True)
 707         self.assertEquals([], tests_run)
 708
 709     def test_reftest_driver_should_run_expected_html(self):
 710         tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True, include_reference_html=True)
 711         self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run)
 712
 713     def test_reftest_driver_should_run_expected_mismatch_html(self):
 714         tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True, include_reference_html=True)
 715         self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run)
 716
 717     def test_additional_platform_directory(self):
 718         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
 719         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
 720         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', '--additional-platform-directory', '/tmp/bar']))
 721
 722         res, buildbot_output, regular_output, user = logging_run(['--additional-platform-directory', 'foo'])
 723         self.assertTrue('--additional-platform-directory=foo is ignored since it is not absolute\n' in regular_output.get())
 724
 725     def test_no_http_and_force(self):
 726         # See test_run_force, using --force raises an exception.
 727         # FIXME: We would like to check the warnings generated.
 728         self.assertRaises(ValueError, logging_run, ['--force', '--no-http'])
 729
 730     @staticmethod
 731     def has_test_of_type(tests, type):
 732         return [test for test in tests if type in test]
 733
 734     def test_no_http_tests(self):
 735         batch_tests_dryrun = get_tests_run(['LayoutTests/http', 'websocket/'], flatten_batches=True)
 736         self.assertTrue(MainTest.has_test_of_type(batch_tests_dryrun, 'http'))
 737         self.assertTrue(MainTest.has_test_of_type(batch_tests_dryrun, 'websocket'))
 738
 739         batch_tests_run_no_http = get_tests_run(['--no-http', 'LayoutTests/http', 'websocket/'], flatten_batches=True)
 740         self.assertFalse(MainTest.has_test_of_type(batch_tests_run_no_http, 'http'))
 741         self.assertFalse(MainTest.has_test_of_type(batch_tests_run_no_http, 'websocket'))
 742
 743         batch_tests_run_http = get_tests_run(['--http', 'LayoutTests/http', 'websocket/'], flatten_batches=True)
 744         self.assertTrue(MainTest.has_test_of_type(batch_tests_run_http, 'http'))
 745         self.assertTrue(MainTest.has_test_of_type(batch_tests_run_http, 'websocket'))
 746
 747 MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
 748
 749
 750 class EndToEndTest(unittest.TestCase):
 751     def parse_full_results(self, full_results_text):
 752         json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
 753         compressed_results = json.loads(json_to_eval)
 754         return compressed_results
 755
 756     def test_end_to_end(self):
 757         fs = unit_test_filesystem()
 758         res, out, err, user = logging_run(record_results=True, tests_included=True, filesystem=fs)
 759
 760         # Seven tests should fail, so the return code should be 7.
 761         self.assertEquals(res, 7)
 762         results = self.parse_full_results(fs.files['/tmp/layout-test-results/full_results.json'])
 763
 764         # Check to ensure we're passing back image diff %age correctly.
 765         self.assertEquals(results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1)
 766
 767         # Check that we attempted to display the results page in a browser.
 768         self.assertTrue(user.opened_urls)
 769
 770 class RebaselineTest(unittest.TestCase):
 771     def assertBaselines(self, file_list, file, extensions, err):
 772         "assert that the file_list contains the baselines."""
 773         for ext in extensions:
 774             baseline = file + "-expected" + ext
 775             baseline_msg = 'Writing new expected result "%s"\n' % baseline[1:]
 776             self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
 777             self.assertTrue(baseline_msg in err.get())
 778
 779     # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
 780     # supposed to be.
 781
 782     def test_reset_results(self):
 783         # Test that we update expectations in place. If the expectation
 784         # is missing, update the expected generic location.
 785         fs = unit_test_filesystem()
 786         res, out, err, _ = logging_run(['--pixel-tests',
 787                         '--reset-results',
 788                         'passes/image.html',
 789                         'failures/expected/missing_image.html'],
 790                         tests_included=True, filesystem=fs, new_results=True)
 791         file_list = fs.written_files.keys()
 792         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
 793         self.assertEquals(res, 0)
 794         self.assertTrue(out.empty())
 795         self.assertEqual(len(file_list), 4)
 796         self.assertBaselines(file_list, "/passes/image", [".txt", ".png"], err)
 797         self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"], err)
 798
 799     def test_missing_results(self):
 800         # Test that we update expectations in place. If the expectation
 801         # is missing, update the expected generic location.
 802         fs = unit_test_filesystem()
 803         res, out, err, _ = logging_run(['--no-show-results',
 804                      'failures/unexpected/missing_text.html',
 805                      'failures/unexpected/missing_image.html',
 806                      'failures/unexpected/missing_audio.html',
 807                      'failures/unexpected/missing_render_tree_dump.html'],
 808                      tests_included=True, filesystem=fs, new_results=True)
 809         file_list = fs.written_files.keys()
 810         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
 811         self.assertEquals(res, 0)
 812         self.assertFalse(out.empty())
 813         self.assertEqual(len(file_list), 6)
 814         self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"], err)
 815         self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_image", [".png"], err)
 816         self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_render_tree_dump", [".txt"], err)
 817
 818     def test_new_baseline(self):
 819         # Test that we update the platform expectations. If the expectation
 820         # is mssing, then create a new expectation in the platform dir.
 821         fs = unit_test_filesystem()
 822         res, out, err, _ = logging_run(['--pixel-tests',
 823                         '--new-baseline',
 824                         'passes/image.html',
 825                         'failures/expected/missing_image.html'],
 826                     tests_included=True, filesystem=fs, new_results=True)
 827         file_list = fs.written_files.keys()
 828         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
 829         self.assertEquals(res, 0)
 830         self.assertTrue(out.empty())
 831         self.assertEqual(len(file_list), 4)
 832         self.assertBaselines(file_list,
 833             "/platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
 834         self.assertBaselines(file_list,
 835             "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)
 836
 837
 838 class DryrunTest(unittest.TestCase):
 839     # FIXME: it's hard to know which platforms are safe to test; the
 840     # chromium platforms require a chromium checkout, and the mac platform
 841     # requires fcntl, so it can't be tested on win32, etc. There is
 842     # probably a better way of handling this.
 843     def disabled_test_darwin(self):
 844         if sys.platform != "darwin":
 845             return
 846
 847         self.assertTrue(passing_run(['--platform', 'dryrun', 'fast/html'], tests_included=True))
 848         self.assertTrue(passing_run(['--platform', 'dryrun-mac', 'fast/html'], tests_included=True))
 849
 850     def test_test(self):
 851         self.assertTrue(passing_run(['--platform', 'dryrun-test', '--pixel-tests']))
 852
 853
 854 if __name__ == '__main__':
 855     unittest.main()