src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py

   1 # Copyright (C) 2010, 2012 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 """Unit tests for printing.py."""
  30
  31 import StringIO
  32 import optparse
  33 import sys
  34 import time
  35 import unittest
  36
  37 from webkitpy.common.host_mock import MockHost
  38
  39 from webkitpy.common.system import logtesting
  40 from webkitpy.layout_tests import port
  41 from webkitpy.layout_tests.controllers import manager
  42 from webkitpy.layout_tests.models import test_expectations
  43 from webkitpy.layout_tests.models import test_failures
  44 from webkitpy.layout_tests.models import test_results
  45 from webkitpy.layout_tests.views import printing
  46
  47
  48 def get_options(args):
  49     print_options = printing.print_options()
  50     option_parser = optparse.OptionParser(option_list=print_options)
  51     return option_parser.parse_args(args)
  52
  53
  54 class TestUtilityFunctions(unittest.TestCase):
  55     def test_print_options(self):
  56         options, args = get_options([])
  57         self.assertIsNotNone(options)
  58
  59
  60 class FakeRunResults(object):
  61     def __init__(self, total=1, expected=1, unexpected=0, fake_results=None):
  62         fake_results = fake_results or []
  63         self.total = total
  64         self.expected = expected
  65         self.expected_failures = 0
  66         self.unexpected = unexpected
  67         self.expected_skips = 0
  68         self.results_by_name = {}
  69         total_run_time = 0
  70         for result in fake_results:
  71             self.results_by_name[result.shard_name] = result
  72             total_run_time += result.total_run_time
  73         self.run_time = total_run_time + 1
  74
  75
  76 class FakeShard(object):
  77     def __init__(self, shard_name, total_run_time):
  78         self.shard_name = shard_name
  79         self.total_run_time = total_run_time
  80
  81
  82 class  Testprinter(unittest.TestCase):
  83     def assertEmpty(self, stream):
  84         self.assertFalse(stream.getvalue())
  85
  86     def assertNotEmpty(self, stream):
  87         self.assertTrue(stream.getvalue())
  88
  89     def assertWritten(self, stream, contents):
  90         self.assertEqual(stream.buflist, contents)
  91
  92     def reset(self, stream):
  93         stream.buflist = []
  94         stream.buf = ''
  95
  96     def get_printer(self, args=None):
  97         args = args or []
  98         printing_options = printing.print_options()
  99         option_parser = optparse.OptionParser(option_list=printing_options)
 100         options, args = option_parser.parse_args(args)
 101         host = MockHost()
 102         self._port = host.port_factory.get('test', options)
 103         nproc = 2
 104
 105         regular_output = StringIO.StringIO()
 106         printer = printing.Printer(self._port, options, regular_output)
 107         return printer, regular_output
 108
 109     def get_result(self, test_name, result_type=test_expectations.PASS, run_time=0):
 110         failures = []
 111         if result_type == test_expectations.TIMEOUT:
 112             failures = [test_failures.FailureTimeout()]
 113         elif result_type == test_expectations.CRASH:
 114             failures = [test_failures.FailureCrash()]
 115         return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
 116
 117     def test_configure_and_cleanup(self):
 118         # This test verifies that calling cleanup repeatedly and deleting
 119         # the object is safe.
 120         printer, err = self.get_printer()
 121         printer.cleanup()
 122         printer.cleanup()
 123         printer = None
 124
 125     def test_print_config(self):
 126         printer, err = self.get_printer()
 127         # FIXME: it's lame that i have to set these options directly.
 128         printer._options.pixel_tests = True
 129         printer._options.new_baseline = True
 130         printer._options.time_out_ms = 6000
 131         printer._options.slow_time_out_ms = 12000
 132         printer.print_config('/tmp')
 133         self.assertIn("Using port 'test-mac-leopard'", err.getvalue())
 134         self.assertIn('Test configuration: <leopard, x86, release>', err.getvalue())
 135         self.assertIn('View the test results at file:///tmp', err.getvalue())
 136         self.assertIn('View the archived results dashboard at file:///tmp', err.getvalue())
 137         self.assertIn('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic', err.getvalue())
 138         self.assertIn('Using Release build', err.getvalue())
 139         self.assertIn('Pixel tests enabled', err.getvalue())
 140         self.assertIn('Command line:', err.getvalue())
 141         self.assertIn('Regular timeout: ', err.getvalue())
 142
 143         self.reset(err)
 144         printer._options.quiet = True
 145         printer.print_config('/tmp')
 146         self.assertNotIn('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic', err.getvalue())
 147
 148     def test_print_directory_timings(self):
 149         printer, err = self.get_printer()
 150         printer._options.debug_rwt_logging = True
 151
 152         run_results = FakeRunResults()
 153         run_results.results_by_name = {
 154             "slowShard": FakeShard("slowShard", 16),
 155             "borderlineShard": FakeShard("borderlineShard", 15),
 156             "fastShard": FakeShard("fastShard", 1),
 157         }
 158
 159         printer._print_directory_timings(run_results)
 160         self.assertWritten(err, ['Time to process slowest subdirectories:\n', '  slowShard took 16.0 seconds to run 1 tests.\n', '\n'])
 161
 162         printer, err = self.get_printer()
 163         printer._options.debug_rwt_logging = True
 164
 165         run_results.results_by_name = {
 166             "borderlineShard": FakeShard("borderlineShard", 15),
 167             "fastShard": FakeShard("fastShard", 1),
 168         }
 169
 170         printer._print_directory_timings(run_results)
 171         self.assertWritten(err, [])
 172
 173     def test_print_one_line_summary(self):
 174         def run_test(total, exp, unexp, shards, result):
 175             printer, err = self.get_printer(['--timing'] if shards else None)
 176             fake_results = FakeRunResults(total, exp, unexp, shards)
 177             total_time = fake_results.run_time + 1
 178             printer._print_one_line_summary(total_time, fake_results)
 179             self.assertWritten(err, result)
 180
 181         # Without times:
 182         run_test(1, 1, 0, [], ["The test ran as expected.\n", "\n"])
 183         run_test(2, 1, 1, [], ["\n", "1 test ran as expected, 1 didn't:\n", "\n"])
 184         run_test(3, 2, 1, [], ["\n", "2 tests ran as expected, 1 didn't:\n", "\n"])
 185         run_test(3, 2, 0, [], ["\n", "2 tests ran as expected (1 didn't run).\n", "\n"])
 186
 187         # With times:
 188         fake_shards = [FakeShard("foo", 1), FakeShard("bar", 2)]
 189         run_test(1, 1, 0, fake_shards, ["The test ran as expected in 5.00s (2.00s in rwt, 1x).\n", "\n"])
 190         run_test(2, 1, 1, fake_shards, ["\n", "1 test ran as expected, 1 didn't in 5.00s (2.00s in rwt, 1x):\n", "\n"])
 191         run_test(3, 2, 1, fake_shards, ["\n", "2 tests ran as expected, 1 didn't in 5.00s (2.00s in rwt, 1x):\n", "\n"])
 192         run_test(3, 2, 0, fake_shards, ["\n", "2 tests ran as expected (1 didn't run) in 5.00s (2.00s in rwt, 1x).\n", "\n"])
 193
 194     def test_test_status_line(self):
 195         printer, _ = self.get_printer()
 196         printer._meter.number_of_columns = lambda: 80
 197         actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
 198         self.assertEqual(80, len(actual))
 199         self.assertEqual(actual, '[0/0] fast/dom/HTMLFormElement/associa...after-index-assertion-fail1.html passed')
 200
 201         printer._meter.number_of_columns = lambda: 89
 202         actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
 203         self.assertEqual(89, len(actual))
 204         self.assertEqual(actual, '[0/0] fast/dom/HTMLFormElement/associated-...ents-after-index-assertion-fail1.html passed')
 205
 206         printer._meter.number_of_columns = lambda: sys.maxint
 207         actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
 208         self.assertEqual(90, len(actual))
 209         self.assertEqual(actual, '[0/0] fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html passed')
 210
 211         printer._meter.number_of_columns = lambda: 18
 212         actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
 213         self.assertEqual(18, len(actual))
 214         self.assertEqual(actual, '[0/0] f...l passed')
 215
 216         printer._meter.number_of_columns = lambda: 10
 217         actual = printer._test_status_line('fast/dom/HTMLFormElement/associated-elements-after-index-assertion-fail1.html', ' passed')
 218         self.assertEqual(actual, '[0/0] associated-elements-after-index-assertion-fail1.html passed')
 219
 220     def test_details(self):
 221         printer, err = self.get_printer(['--details'])
 222         result = self.get_result('passes/image.html')
 223         printer.print_started_test('passes/image.html')
 224         printer.print_finished_test(result, expected=False, exp_str='', got_str='')
 225         self.assertNotEmpty(err)
 226
 227     def test_print_found(self):
 228         printer, err = self.get_printer()
 229
 230         printer.print_found(100, 10, 1, 1)
 231         self.assertWritten(err, ["Found 100 tests; running 10, skipping 90.\n"])
 232
 233         self.reset(err)
 234         printer.print_found(100, 10, 2, 3)
 235         self.assertWritten(err, ["Found 100 tests; running 10 (6 times each: --repeat-each=2 --iterations=3), skipping 90.\n"])
 236
 237     def test_debug_rwt_logging_is_throttled(self):
 238         printer, err = self.get_printer(['--debug-rwt-logging'])
 239
 240         result = self.get_result('passes/image.html')
 241         printer.print_started_test('passes/image.html')
 242         printer.print_finished_test(result, expected=True, exp_str='', got_str='')
 243
 244         printer.print_started_test('passes/text.html')
 245         result = self.get_result('passes/text.html')
 246         printer.print_finished_test(result, expected=True, exp_str='', got_str='')
 247
 248         # Only the first test's start should be printed.
 249         lines = err.buflist
 250         self.assertEqual(len(lines), 1)
 251         self.assertTrue(lines[0].endswith('passes/image.html\n'))