src/third_party/WebKit/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #    * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #    * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #    * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import unittest
  30
  31 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer
  32 from webkitpy.common.checkout.scm.scm_mock import MockSCM
  33 from webkitpy.common.host_mock import MockHost
  34 from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder
  35 from webkitpy.common.net.layouttestresults import LayoutTestResults
  36 from webkitpy.common.system.executive_mock import MockExecutive
  37 from webkitpy.common.system.executive_mock import MockExecutive2
  38 from webkitpy.common.system.outputcapture import OutputCapture
  39 from webkitpy.tool.commands.rebaseline import *
  40 from webkitpy.tool.mocktool import MockTool, MockOptions
  41
  42
  43 class _BaseTestCase(unittest.TestCase):
  44     MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True'
  45     WEB_PREFIX = 'http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results'
  46
  47     command_constructor = None
  48
  49     def setUp(self):
  50         self.tool = MockTool()
  51         self.command = self.command_constructor()  # lint warns that command_constructor might not be set, but this is intentional; pylint: disable=E1102
  52         self.command.bind_to_tool(self.tool)
  53         self.lion_port = self.tool.port_factory.get_from_builder_name("WebKit Mac10.7")
  54         self.lion_expectations_path = self.lion_port.path_to_generic_test_expectations_file()
  55         self.tool.filesystem.write_text_file(self.tool.filesystem.join(self.lion_port.layout_tests_dir(), "VirtualTestSuites"),
  56                                              '[]')
  57
  58         # FIXME: crbug.com/279494. We should override builders._exact_matches
  59         # here to point to a set of test ports and restore the value in
  60         # tearDown(), and that way the individual tests wouldn't have to worry
  61         # about it.
  62
  63     def _expand(self, path):
  64         if self.tool.filesystem.isabs(path):
  65             return path
  66         return self.tool.filesystem.join(self.lion_port.layout_tests_dir(), path)
  67
  68     def _read(self, path):
  69         return self.tool.filesystem.read_text_file(self._expand(path))
  70
  71     def _write(self, path, contents):
  72         self.tool.filesystem.write_text_file(self._expand(path), contents)
  73
  74     def _zero_out_test_expectations(self):
  75         for port_name in self.tool.port_factory.all_port_names():
  76             port = self.tool.port_factory.get(port_name)
  77             for path in port.expectations_files():
  78                 self._write(path, '')
  79         self.tool.filesystem.written_files = {}
  80
  81     def _setup_mock_builder_data(self):
  82         data = LayoutTestResults.results_from_string("""ADD_RESULTS({
  83     "tests": {
  84         "userscripts": {
  85             "first-test.html": {
  86                 "expected": "PASS",
  87                 "actual": "IMAGE+TEXT"
  88             },
  89             "second-test.html": {
  90                 "expected": "FAIL",
  91                 "actual": "IMAGE+TEXT"
  92             }
  93         }
  94     }
  95 });""")
  96         # FIXME: crbug.com/279494 - we shouldn't be mixing mock and real builder names.
  97         for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7']:
  98             self.command._builder_data[builder] = data
  99
 100
 101 class TestCopyExistingBaselinesInternal(_BaseTestCase):
 102     command_constructor = CopyExistingBaselinesInternal
 103
 104     def setUp(self):
 105         super(TestCopyExistingBaselinesInternal, self).setUp()
 106
 107     def test_copying_overwritten_baseline(self):
 108         self.tool.executive = MockExecutive2()
 109
 110         # FIXME: crbug.com/279494. it's confusing that this is the test- port, and not the regular lion port. Really all of the tests should be using the test ports.
 111         port = self.tool.port_factory.get('test-mac-snowleopard')
 112         self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-mac-snowleopard/failures/expected/image-expected.txt'), 'original snowleopard result')
 113
 114         old_exact_matches = builders._exact_matches
 115         oc = OutputCapture()
 116         try:
 117             builders._exact_matches = {
 118                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 119                 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 120             }
 121
 122             options = MockOptions(builder="MOCK SnowLeopard", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None)
 123
 124             oc.capture_output()
 125             self.command.execute(options, [], self.tool)
 126         finally:
 127             out, _, _ = oc.restore_output()
 128             builders._exact_matches = old_exact_matches
 129
 130         self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-mac-leopard/failures/expected/image-expected.txt')), 'original snowleopard result')
 131         self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n')
 132
 133     def test_copying_overwritten_baseline_to_multiple_locations(self):
 134         self.tool.executive = MockExecutive2()
 135
 136         # FIXME: crbug.com/279494. it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports.
 137         port = self.tool.port_factory.get('test-win-win7')
 138         self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result')
 139
 140         old_exact_matches = builders._exact_matches
 141         oc = OutputCapture()
 142         try:
 143             builders._exact_matches = {
 144                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 145                 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])},
 146                 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])},
 147             }
 148
 149             options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None)
 150
 151             oc.capture_output()
 152             self.command.execute(options, [], self.tool)
 153         finally:
 154             out, _, _ = oc.restore_output()
 155             builders._exact_matches = old_exact_matches
 156
 157         self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result')
 158         self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt')))
 159         self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n')
 160
 161     def test_no_copy_existing_baseline(self):
 162         self.tool.executive = MockExecutive2()
 163
 164         # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports.
 165         port = self.tool.port_factory.get('test-win-win7')
 166         self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result')
 167
 168         old_exact_matches = builders._exact_matches
 169         oc = OutputCapture()
 170         try:
 171             builders._exact_matches = {
 172                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 173                 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])},
 174                 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])},
 175             }
 176
 177             options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None)
 178
 179             oc.capture_output()
 180             self.command.execute(options, [], self.tool)
 181         finally:
 182             out, _, _ = oc.restore_output()
 183             builders._exact_matches = old_exact_matches
 184
 185         self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result')
 186         self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'original win7 result')
 187         self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt')))
 188         self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n')
 189
 190     def test_no_copy_skipped_test(self):
 191         self.tool.executive = MockExecutive2()
 192
 193         port = self.tool.port_factory.get('test-win-win7')
 194         fs = self.tool.filesystem
 195         self._write(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result')
 196         expectations_path = fs.join(port.path_to_generic_test_expectations_file())
 197         self._write(expectations_path, (
 198             "[ Win ] failures/expected/image.html [ Failure ]\n"
 199             "[ Linux ] failures/expected/image.html [ Skip ]\n"))
 200         old_exact_matches = builders._exact_matches
 201         oc = OutputCapture()
 202         try:
 203             builders._exact_matches = {
 204                 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])},
 205                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 206                 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])},
 207             }
 208
 209             options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None)
 210
 211             oc.capture_output()
 212             self.command.execute(options, [], self.tool)
 213         finally:
 214             out, _, _ = oc.restore_output()
 215             builders._exact_matches = old_exact_matches
 216
 217         self.assertFalse(fs.exists(fs.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')))
 218         self.assertEqual(self._read(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')),
 219                          'original win7 result')
 220
 221
 222 class TestRebaselineTest(_BaseTestCase):
 223     command_constructor = RebaselineTest  # AKA webkit-patch rebaseline-test-internal
 224
 225     def setUp(self):
 226         super(TestRebaselineTest, self).setUp()
 227         self.options = MockOptions(builder="WebKit Mac10.7", test="userscripts/another-test.html", suffixes="txt", results_directory=None)
 228
 229     def test_baseline_directory(self):
 230         command = self.command
 231         self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-lion")
 232         self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard")
 233
 234     def test_rebaseline_updates_expectations_file_noop(self):
 235         self._zero_out_test_expectations()
 236         self._write(self.lion_expectations_path, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]
 237 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]
 238 """)
 239         self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.html", "Dummy test contents")
 240         self._write("fast/css/large-list-of-rules-crash.html", "Dummy test contents")
 241         self._write("userscripts/another-test.html", "Dummy test contents")
 242
 243         self.options.suffixes = "png,wav,txt"
 244         self.command._rebaseline_test_and_update_expectations(self.options)
 245
 246         self.assertItemsEqual(self.tool.web.urls_fetched,
 247             [self.WEB_PREFIX + '/userscripts/another-test-actual.png',
 248              self.WEB_PREFIX + '/userscripts/another-test-actual.wav',
 249              self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
 250         new_expectations = self._read(self.lion_expectations_path)
 251         self.assertMultiLineEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ]
 252 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ]
 253 """)
 254
 255     def test_rebaseline_test(self):
 256         self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", self.WEB_PREFIX)
 257         self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt'])
 258
 259     def test_rebaseline_test_with_results_directory(self):
 260         self._write("userscripts/another-test.html", "test data")
 261         self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n")
 262         self.options.results_directory = '/tmp'
 263         self.command._rebaseline_test_and_update_expectations(self.options)
 264         self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscripts/another-test-actual.txt'])
 265
 266     def test_rebaseline_reftest(self):
 267         self._write("userscripts/another-test.html", "test data")
 268         self._write("userscripts/another-test-expected.html", "generic result")
 269         OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_update_expectations, args=[self.options],
 270             expected_logs="Cannot rebaseline reftest: userscripts/another-test.html\n")
 271         self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-lines': [], "delete": []})
 272
 273     def test_rebaseline_test_and_print_scm_changes(self):
 274         self.command._print_scm_changes = True
 275         self.command._scm_changes = {'add': [], 'delete': []}
 276         self.tool._scm.exists = lambda x: False
 277
 278         self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", None)
 279
 280         self.assertDictEqual(self.command._scm_changes, {'add': ['/mock-checkout/third_party/WebKit/LayoutTests/platform/linux/userscripts/another-test-expected.txt'], 'delete': []})
 281
 282     def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self):
 283         self.tool.executive = MockExecutive2()
 284
 285         # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports.
 286         port = self.tool.port_factory.get('test-win-win7')
 287         self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result')
 288
 289         old_exact_matches = builders._exact_matches
 290         oc = OutputCapture()
 291         try:
 292             builders._exact_matches = {
 293                 "MOCK XP": {"port_name": "test-win-xp"},
 294                 "MOCK Win7": {"port_name": "test-win-win7"},
 295             }
 296
 297             options = MockOptions(optimize=True, builder="MOCK Win7", suffixes="txt",
 298                 verbose=True, test="failures/expected/image.html", results_directory=None)
 299
 300             oc.capture_output()
 301             self.command.execute(options, [], self.tool)
 302         finally:
 303             out, _, _ = oc.restore_output()
 304             builders._exact_matches = old_exact_matches
 305
 306         self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True')
 307         self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-xp/failures/expected/image-expected.txt')))
 308         self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win7"}], "delete": []}\n')
 309
 310
 311 class TestAbstractParallelRebaselineCommand(_BaseTestCase):
 312     command_constructor = AbstractParallelRebaselineCommand
 313
 314     def test_builders_to_fetch_from(self):
 315         old_exact_matches = builders._exact_matches
 316         try:
 317             builders._exact_matches = {
 318                 "MOCK XP": {"port_name": "test-win-xp"},
 319                 "MOCK Win7": {"port_name": "test-win-win7"},
 320                 "MOCK Win7 (dbg)(1)": {"port_name": "test-win-win7"},
 321                 "MOCK Win7 (dbg)(2)": {"port_name": "test-win-win7"},
 322             }
 323
 324             builders_to_fetch = self.command._builders_to_fetch_from(["MOCK XP", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win7"])
 325             self.assertEqual(builders_to_fetch, ["MOCK XP", "MOCK Win7"])
 326         finally:
 327             builders._exact_matches = old_exact_matches
 328
 329
 330 class TestRebaselineJson(_BaseTestCase):
 331     command_constructor = RebaselineJson
 332
 333     def setUp(self):
 334         super(TestRebaselineJson, self).setUp()
 335         self.tool.executive = MockExecutive2()
 336         self.old_exact_matches = builders._exact_matches
 337         builders._exact_matches = {
 338             "MOCK builder": {"port_name": "test-mac-snowleopard"},
 339             "MOCK builder (Debug)": {"port_name": "test-mac-snowleopard"},
 340         }
 341
 342     def tearDown(self):
 343         builders._exact_matches = self.old_exact_matches
 344         super(TestRebaselineJson, self).tearDown()
 345
 346     def test_rebaseline_test_passes_on_all_builders(self):
 347         self._setup_mock_builder_data()
 348
 349         def builder_data():
 350             self.command._builder_data['MOCK builder'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
 351     "tests": {
 352         "userscripts": {
 353             "first-test.html": {
 354                 "expected": "NEEDSREBASELINE",
 355                 "actual": "PASS"
 356             }
 357         }
 358     }
 359 });""")
 360             return self.command._builder_data
 361
 362         self.command.builder_data = builder_data
 363
 364         options = MockOptions(optimize=True, verbose=True, results_directory=None)
 365
 366         self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n")
 367         self._write("userscripts/first-test.html", "Dummy test contents")
 368
 369         self.command._rebaseline(options,  {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}})
 370
 371         # Note that we have one run_in_parallel() call followed by a run_command()
 372         self.assertEqual(self.tool.executive.calls,
 373             [[['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', '', 'userscripts/first-test.html', '--verbose']]])
 374
 375     def test_rebaseline_all(self):
 376         self._setup_mock_builder_data()
 377
 378         options = MockOptions(optimize=True, verbose=True, results_directory=None)
 379         self._write("userscripts/first-test.html", "Dummy test contents")
 380         self.command._rebaseline(options,  {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}})
 381
 382         # Note that we have one run_in_parallel() call followed by a run_command()
 383         self.assertEqual(self.tool.executive.calls,
 384             [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']],
 385              [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']],
 386              [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]])
 387
 388     def test_rebaseline_debug(self):
 389         self._setup_mock_builder_data()
 390
 391         options = MockOptions(optimize=True, verbose=True, results_directory=None)
 392         self._write("userscripts/first-test.html", "Dummy test contents")
 393         self.command._rebaseline(options,  {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
 394
 395         # Note that we have one run_in_parallel() call followed by a run_command()
 396         self.assertEqual(self.tool.executive.calls,
 397             [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']],
 398              [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']],
 399              [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]])
 400
 401     def test_no_optimize(self):
 402         self._setup_mock_builder_data()
 403
 404         options = MockOptions(optimize=False, verbose=True, results_directory=None)
 405         self._write("userscripts/first-test.html", "Dummy test contents")
 406         self.command._rebaseline(options,  {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}})
 407
 408         # Note that we have only one run_in_parallel() call
 409         self.assertEqual(self.tool.executive.calls,
 410             [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']],
 411              [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']]])
 412
 413     def test_results_directory(self):
 414         self._setup_mock_builder_data()
 415
 416         options = MockOptions(optimize=False, verbose=True, results_directory='/tmp')
 417         self._write("userscripts/first-test.html", "Dummy test contents")
 418         self.command._rebaseline(options,  {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}})
 419
 420         # Note that we have only one run_in_parallel() call
 421         self.assertEqual(self.tool.executive.calls,
 422             [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']],
 423              [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']]])
 424
 425 class TestRebaselineJsonUpdatesExpectationsFiles(_BaseTestCase):
 426     command_constructor = RebaselineJson
 427
 428     def setUp(self):
 429         super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp()
 430         self.tool.executive = MockExecutive2()
 431
 432         def mock_run_command(args,
 433                              cwd=None,
 434                              input=None,
 435                              error_handler=None,
 436                              return_exit_code=False,
 437                              return_stderr=True,
 438                              decode_output=False,
 439                              env=None):
 440             return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "builder": "WebKit Mac10.7"}]}\n'
 441         self.tool.executive.run_command = mock_run_command
 442
 443     def test_rebaseline_updates_expectations_file(self):
 444         options = MockOptions(optimize=False, verbose=True, results_directory=None)
 445
 446         self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
 447         self._write("userscripts/first-test.html", "Dummy test contents")
 448         self._setup_mock_builder_data()
 449
 450         self.command._rebaseline(options,  {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
 451
 452         new_expectations = self._read(self.lion_expectations_path)
 453         self.assertMultiLineEqual(new_expectations, "Bug(x) [ Mavericks MountainLion Retina SnowLeopard ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
 454
 455     def test_rebaseline_updates_expectations_file_all_platforms(self):
 456         options = MockOptions(optimize=False, verbose=True, results_directory=None)
 457
 458         self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n")
 459         self._write("userscripts/first-test.html", "Dummy test contents")
 460         self._setup_mock_builder_data()
 461
 462         self.command._rebaseline(options,  {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
 463
 464         new_expectations = self._read(self.lion_expectations_path)
 465         self.assertMultiLineEqual(new_expectations, "Bug(x) [ Android Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
 466
 467     def test_rebaseline_handles_platform_skips(self):
 468         # This test is just like test_rebaseline_updates_expectations_file_all_platforms(),
 469         # except that if a particular port happens to SKIP a test in an overrides file,
 470         # we count that as passing, and do not think that we still need to rebaseline it.
 471         options = MockOptions(optimize=False, verbose=True, results_directory=None)
 472
 473         self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n")
 474         self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ Skip ]\n")
 475         self._write("userscripts/first-test.html", "Dummy test contents")
 476         self._setup_mock_builder_data()
 477
 478         self.command._rebaseline(options,  {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
 479
 480         new_expectations = self._read(self.lion_expectations_path)
 481         self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
 482
 483     def test_rebaseline_handles_skips_in_file(self):
 484         # This test is like test_Rebaseline_handles_platform_skips, except that the
 485         # Skip is in the same (generic) file rather than a platform file. In this case,
 486         # the Skip line should be left unmodified. Note that the first line is now
 487         # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with
 488         # the second line.
 489         options = MockOptions(optimize=False, verbose=True, results_directory=None)
 490
 491         self._write(self.lion_expectations_path,
 492             ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n"
 493              "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"))
 494         self._write("userscripts/first-test.html", "Dummy test contents")
 495         self._setup_mock_builder_data()
 496
 497         self.command._rebaseline(options,  {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
 498
 499         new_expectations = self._read(self.lion_expectations_path)
 500         self.assertMultiLineEqual(new_expectations,
 501             ("Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n"
 502              "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n"))
 503
 504     def test_rebaseline_handles_smoke_tests(self):
 505         # This test is just like test_rebaseline_handles_platform_skips, except that we check for
 506         # a test not being in the SmokeTests file, instead of using overrides files.
 507         # If a test is not part of the smoke tests, we count that as passing on ports that only
 508         # run smoke tests, and do not think that we still need to rebaseline it.
 509         options = MockOptions(optimize=False, verbose=True, results_directory=None)
 510
 511         self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n")
 512         self._write("SmokeTests", "fast/html/article-element.html")
 513         self._write("userscripts/first-test.html", "Dummy test contents")
 514         self._setup_mock_builder_data()
 515
 516         self.command._rebaseline(options,  {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}})
 517
 518         new_expectations = self._read(self.lion_expectations_path)
 519         self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n")
 520
 521
 522 class TestRebaseline(_BaseTestCase):
 523     # This command shares most of its logic with RebaselineJson, so these tests just test what is different.
 524
 525     command_constructor = Rebaseline  # AKA webkit-patch rebaseline
 526
 527     def test_rebaseline(self):
 528         self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')]
 529
 530         self._write("userscripts/first-test.html", "test data")
 531
 532         self._zero_out_test_expectations()
 533         self._setup_mock_builder_data()
 534
 535         old_exact_matches = builders._exact_matches
 536         try:
 537             builders._exact_matches = {
 538                 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 539             }
 540             self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts/first-test.html'], self.tool)
 541         finally:
 542             builders._exact_matches = old_exact_matches
 543
 544         calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls)
 545         self.assertEqual(calls,
 546             [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']],
 547              [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']]])
 548
 549     def test_rebaseline_directory(self):
 550         self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')]
 551
 552         self._write("userscripts/first-test.html", "test data")
 553         self._write("userscripts/second-test.html", "test data")
 554
 555         self._setup_mock_builder_data()
 556
 557         old_exact_matches = builders._exact_matches
 558         try:
 559             builders._exact_matches = {
 560                 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 561             }
 562             self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts'], self.tool)
 563         finally:
 564             builders._exact_matches = old_exact_matches
 565
 566         calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls)
 567         self.assertEqual(calls,
 568             [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'],
 569               ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']],
 570              [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'],
 571               ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']]])
 572
 573
 574 class MockLineRemovingExecutive(MockExecutive):
 575     def run_in_parallel(self, commands):
 576         assert len(commands)
 577
 578         num_previous_calls = len(self.calls)
 579         command_outputs = []
 580         for cmd_line, cwd in commands:
 581             out = self.run_command(cmd_line, cwd=cwd)
 582             if 'rebaseline-test-internal' in cmd_line:
 583                 out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6])
 584             command_outputs.append([0, out, ''])
 585
 586         new_calls = self.calls[num_previous_calls:]
 587         self.calls = self.calls[:num_previous_calls]
 588         self.calls.append(new_calls)
 589         return command_outputs
 590
 591
 592 class TestRebaselineExpectations(_BaseTestCase):
 593     command_constructor = RebaselineExpectations
 594
 595     def setUp(self):
 596         super(TestRebaselineExpectations, self).setUp()
 597         self.options = MockOptions(optimize=False, builders=None, suffixes=['txt'], verbose=False, platform=None, results_directory=None)
 598
 599     def _write_test_file(self, port, path, contents):
 600         abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path)
 601         self.tool.filesystem.write_text_file(abs_path, contents)
 602
 603     def _setup_test_port(self):
 604         test_port = self.tool.port_factory.get('test')
 605         original_get = self.tool.port_factory.get
 606
 607         def get_test_port(port_name=None, options=None, **kwargs):
 608             if not port_name:
 609                 return test_port
 610             return original_get(port_name, options, **kwargs)
 611         # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path.
 612         # FIXME: crbug.com/279494 - we shouldn't be doing this.
 613         self.tool.port_factory.get = get_test_port
 614
 615         return test_port
 616
 617     def test_rebaseline_expectations(self):
 618         self._zero_out_test_expectations()
 619
 620         self.tool.executive = MockExecutive2()
 621
 622         def builder_data():
 623             self.command._builder_data['MOCK SnowLeopard'] = self.command._builder_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
 624     "tests": {
 625         "userscripts": {
 626             "another-test.html": {
 627                 "expected": "PASS",
 628                 "actual": "PASS TEXT"
 629             },
 630             "images.svg": {
 631                 "expected": "FAIL",
 632                 "actual": "IMAGE+TEXT"
 633             }
 634         }
 635     }
 636 });""")
 637             return self.command._builder_data
 638
 639         self.command.builder_data = builder_data
 640
 641         self._write("userscripts/another-test.html", "Dummy test contents")
 642         self._write("userscripts/images.svg", "Dummy test contents")
 643         self.command._tests_to_rebaseline = lambda port: {
 644             'userscripts/another-test.html': set(['txt']),
 645             'userscripts/images.svg': set(['png']),
 646             'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']),
 647         }
 648
 649         old_exact_matches = builders._exact_matches
 650         try:
 651             builders._exact_matches = {
 652                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 653                 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 654             }
 655             self.command.execute(self.options, [], self.tool)
 656         finally:
 657             builders._exact_matches = old_exact_matches
 658
 659         # FIXME: change this to use the test- ports.
 660         calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls)
 661         self.assertEqual(self.tool.executive.calls, [
 662             [
 663                 ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'],
 664                 ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'],
 665                 ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'],
 666                 ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'],
 667             ],
 668             [
 669                 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'],
 670                 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'],
 671                 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'],
 672                 ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'],
 673             ],
 674         ])
 675
 676     def test_rebaseline_expectations_noop(self):
 677         self._zero_out_test_expectations()
 678
 679         oc = OutputCapture()
 680         try:
 681             oc.capture_output()
 682             self.command.execute(self.options, [], self.tool)
 683         finally:
 684             _, _, logs = oc.restore_output()
 685             self.assertEqual(self.tool.filesystem.written_files, {})
 686             self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n')
 687
 688     def disabled_test_overrides_are_included_correctly(self):
 689         # This tests that the any tests marked as REBASELINE in the overrides are found, but
 690         # that the overrides do not get written into the main file.
 691         self._zero_out_test_expectations()
 692
 693         self._write(self.lion_expectations_path, '')
 694         self.lion_port.expectations_dict = lambda: {
 695             self.lion_expectations_path: '',
 696             'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n'
 697                           'Bug(y) userscripts/test.html [ Crash ]\n')}
 698         self._write('/userscripts/another-test.html', '')
 699
 700         self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': set(['png', 'txt', 'wav'])})
 701         self.assertEqual(self._read(self.lion_expectations_path), '')
 702
 703     def test_rebaseline_without_other_expectations(self):
 704         self._write("userscripts/another-test.html", "Dummy test contents")
 705         self._write(self.lion_expectations_path, "Bug(x) userscripts/another-test.html [ Rebaseline ]\n")
 706         self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': ('png', 'wav', 'txt')})
 707
 708     def test_rebaseline_test_passes_everywhere(self):
 709         test_port = self._setup_test_port()
 710
 711         old_builder_data = self.command.builder_data
 712
 713         def builder_data():
 714             self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
 715     "tests": {
 716         "fast": {
 717             "dom": {
 718                 "prototype-taco.html": {
 719                     "expected": "FAIL",
 720                     "actual": "PASS",
 721                     "is_unexpected": true
 722                 }
 723             }
 724         }
 725     }
 726 });""")
 727             return self.command._builder_data
 728
 729         self.command.builder_data = builder_data
 730
 731         self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
 732 Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ]
 733 """)
 734
 735         self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
 736
 737         self.tool.executive = MockLineRemovingExecutive()
 738
 739         old_exact_matches = builders._exact_matches
 740         try:
 741             builders._exact_matches = {
 742                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 743                 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 744             }
 745
 746             self.command.execute(self.options, [], self.tool)
 747             self.assertEqual(self.tool.executive.calls, [])
 748
 749             # The mac ports should both be removed since they're the only ones in builders._exact_matches.
 750             self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """
 751 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ]
 752 """)
 753         finally:
 754             builders._exact_matches = old_exact_matches
 755
 756
 757 class _FakeOptimizer(BaselineOptimizer):
 758     def read_results_by_directory(self, baseline_name):
 759         if baseline_name.endswith('txt'):
 760             return {'LayoutTests/passes/text.html': '123456'}
 761         return {}
 762
 763
 764 class TestOptimizeBaselines(_BaseTestCase):
 765     command_constructor = OptimizeBaselines
 766
 767     def _write_test_file(self, port, path, contents):
 768         abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path)
 769         self.tool.filesystem.write_text_file(abs_path, contents)
 770
 771     def setUp(self):
 772         super(TestOptimizeBaselines, self).setUp()
 773
 774         # FIXME: This is a hack to get the unittest and the BaselineOptimize to both use /mock-checkout
 775         # instead of one using /mock-checkout and one using /test-checkout.
 776         default_port = self.tool.port_factory.get()
 777         self.tool.port_factory.get = lambda port_name=None: default_port
 778
 779     def test_modify_scm(self):
 780         test_port = self.tool.port_factory.get('test')
 781         self._write_test_file(test_port, 'another/test.html', "Dummy test contents")
 782         self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A")
 783         self._write_test_file(test_port, 'another/test-expected.txt', "result A")
 784
 785         old_exact_matches = builders._exact_matches
 786         try:
 787             builders._exact_matches = {
 788                 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 789             }
 790             OutputCapture().assert_outputs(self, self.command.execute, args=[
 791                 MockOptions(suffixes='txt', no_modify_scm=False, platform='test-mac-snowleopard'),
 792                 ['another/test.html'],
 793                 self.tool,
 794             ], expected_stdout='{"add": [], "remove-lines": [], "delete": []}\n')
 795         finally:
 796             builders._exact_matches = old_exact_matches
 797
 798         self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt')))
 799         self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt')))
 800
 801     def test_no_modify_scm(self):
 802         test_port = self.tool.port_factory.get('test')
 803         self._write_test_file(test_port, 'another/test.html', "Dummy test contents")
 804         self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A")
 805         self._write_test_file(test_port, 'another/test-expected.txt', "result A")
 806
 807         old_exact_matches = builders._exact_matches
 808         try:
 809             builders._exact_matches = {
 810                 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 811             }
 812             OutputCapture().assert_outputs(self, self.command.execute, args=[
 813                 MockOptions(suffixes='txt', no_modify_scm=True, platform='test-mac-snowleopard'),
 814                 ['another/test.html'],
 815                 self.tool,
 816             ], expected_stdout='{"add": [], "remove-lines": [], "delete": ["/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard/another/test-expected.txt"]}\n')
 817         finally:
 818             builders._exact_matches = old_exact_matches
 819
 820         self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt')))
 821         self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt')))
 822
 823     def test_optimize_all_suffixes_by_default(self):
 824         test_port = self.tool.port_factory.get('test')
 825         self._write_test_file(test_port, 'another/test.html', "Dummy test contents")
 826         self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A")
 827         self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.png', "result A png")
 828         self._write_test_file(test_port, 'another/test-expected.txt', "result A")
 829         self._write_test_file(test_port, 'another/test-expected.png', "result A png")
 830
 831         old_exact_matches = builders._exact_matches
 832         try:
 833             builders._exact_matches = {
 834                 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 835             }
 836             oc = OutputCapture()
 837             oc.capture_output()
 838             self.command.execute(MockOptions(suffixes='txt,wav,png', no_modify_scm=True, platform='test-mac-snowleopard'),
 839                                  ['another/test.html'],
 840                                  self.tool)
 841         finally:
 842             out, err, logs = oc.restore_output()
 843             builders._exact_matches = old_exact_matches
 844
 845         self.assertEquals(out, '{"add": [], "remove-lines": [], "delete": ["/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard/another/test-expected.txt", "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard/another/test-expected.png"]}\n')
 846         self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt')))
 847         self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.png')))
 848         self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt')))
 849         self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.png')))
 850
 851
 852 class TestAnalyzeBaselines(_BaseTestCase):
 853     command_constructor = AnalyzeBaselines
 854
 855     def setUp(self):
 856         super(TestAnalyzeBaselines, self).setUp()
 857         self.port = self.tool.port_factory.get('test')
 858         self.tool.port_factory.get = (lambda port_name=None, options=None: self.port)
 859         self.lines = []
 860         self.command._optimizer_class = _FakeOptimizer
 861         self.command._write = (lambda msg: self.lines.append(msg))  # pylint bug warning about unnecessary lambda? pylint: disable=W0108
 862
 863     def test_default(self):
 864         self.command.execute(MockOptions(suffixes='txt', missing=False, platform=None), ['passes/text.html'], self.tool)
 865         self.assertEqual(self.lines,
 866             ['passes/text-expected.txt:',
 867              '  (generic): 123456'])
 868
 869     def test_missing_baselines(self):
 870         self.command.execute(MockOptions(suffixes='png,txt', missing=True, platform=None), ['passes/text.html'], self.tool)
 871         self.assertEqual(self.lines,
 872             ['passes/text-expected.png: (no baselines found)',
 873              'passes/text-expected.txt:',
 874              '  (generic): 123456'])
 875
 876
 877 class TestAutoRebaseline(_BaseTestCase):
 878     command_constructor = AutoRebaseline
 879
 880     def _write_test_file(self, port, path, contents):
 881         abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path)
 882         self.tool.filesystem.write_text_file(abs_path, contents)
 883
 884     def _setup_test_port(self):
 885         test_port = self.tool.port_factory.get('test')
 886         original_get = self.tool.port_factory.get
 887
 888         def get_test_port(port_name=None, options=None, **kwargs):
 889             if not port_name:
 890                 return test_port
 891             return original_get(port_name, options, **kwargs)
 892         # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path.
 893         # FIXME: crbug.com/279494 - we shouldn't be doing this.
 894         self.tool.port_factory.get = get_test_port
 895
 896         return test_port
 897
 898     def setUp(self):
 899         super(TestAutoRebaseline, self).setUp()
 900         self.command.latest_revision_processed_on_all_bots = lambda: 9000
 901         self.command.bot_revision_data = lambda: [{"builder": "Mock builder", "revision": "9000"}]
 902
 903     def test_release_builders(self):
 904         old_exact_matches = builders._exact_matches
 905         try:
 906             builders._exact_matches = {
 907                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
 908                 "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 909                 "MOCK Leopard ASAN": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
 910             }
 911             self.assertEqual(self.command._release_builders(), ['MOCK Leopard'])
 912         finally:
 913             builders._exact_matches = old_exact_matches
 914
 915     def test_tests_to_rebaseline(self):
 916         def blame(path):
 917             return """
 918 624c3081c0 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000   11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ]
 919 624c3081c0 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   13) Bug(foo) path/to/rebaseline-without-bug-number.html [ NeedsRebaseline ]
 920 624c3081c0 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000   11) crbug.com/24182 [ Debug ] path/to/rebaseline-with-modifiers.html [ NeedsRebaseline ]
 921 624c3081c0 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   12) crbug.com/24182 crbug.com/234 path/to/rebaseline-without-modifiers.html [ NeedsRebaseline ]
 922 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   12) crbug.com/24182 path/to/rebaseline-new-revision.html [ NeedsRebaseline ]
 923 624caaaaaa path/to/TestExpectations                   (foo@chromium.org        2013-04-28 04:52:41 +0000   12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]
 924 0000000000 path/to/TestExpectations                   (foo@chromium.org        2013-04-28 04:52:41 +0000   12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]
 925 """
 926         self.tool.scm().blame = blame
 927
 928         min_revision = 9000
 929         self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), (
 930                 set(['path/to/rebaseline-without-bug-number.html', 'path/to/rebaseline-with-modifiers.html', 'path/to/rebaseline-without-modifiers.html']),
 931                 5678,
 932                 'foobarbaz1@chromium.org',
 933                 set(['24182', '234']),
 934                 True))
 935
 936     def test_tests_to_rebaseline_over_limit(self):
 937         def blame(path):
 938             result = ""
 939             for i in range(0, self.command.MAX_LINES_TO_REBASELINE + 1):
 940                 result += "624c3081c0 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   13) crbug.com/24182 path/to/rebaseline-%s.html [ NeedsRebaseline ]\n" % i
 941             return result
 942         self.tool.scm().blame = blame
 943
 944         expected_list_of_tests = []
 945         for i in range(0, self.command.MAX_LINES_TO_REBASELINE):
 946             expected_list_of_tests.append("path/to/rebaseline-%s.html" % i)
 947
 948         min_revision = 9000
 949         self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), (
 950                 set(expected_list_of_tests),
 951                 5678,
 952                 'foobarbaz1@chromium.org',
 953                 set(['24182']),
 954                 True))
 955
 956     def test_commit_message(self):
 957         author = "foo@chromium.org"
 958         revision = 1234
 959         bugs = set()
 960         self.assertEqual(self.command.commit_message(author, revision, bugs),
 961             """Auto-rebaseline for r1234
 962
 963 http://src.chromium.org/viewvc/blink?view=revision&revision=1234
 964
 965 TBR=foo@chromium.org
 966 """)
 967
 968         bugs = set(["234", "345"])
 969         self.assertEqual(self.command.commit_message(author, revision, bugs),
 970             """Auto-rebaseline for r1234
 971
 972 http://src.chromium.org/viewvc/blink?view=revision&revision=1234
 973
 974 BUG=234,345
 975 TBR=foo@chromium.org
 976 """)
 977
 978     def test_no_needs_rebaseline_lines(self):
 979         def blame(path):
 980             return """
 981 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000   11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ]
 982 """
 983         self.tool.scm().blame = blame
 984
 985         self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool)
 986         self.assertEqual(self.tool.executive.calls, [])
 987
 988     def test_execute(self):
 989         def blame(path):
 990             return """
 991 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000   11) # Test NeedsRebaseline being in a comment doesn't bork parsing.
 992 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000   11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ]
 993 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
 994 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000   11) crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ]
 995 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   12) crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ]
 996 624caaaaaa path/to/TestExpectations                   (foo@chromium.org        2013-04-28 04:52:41 +0000   12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]
 997 0000000000 path/to/TestExpectations                   (foo@chromium.org        2013-04-28 04:52:41 +0000   12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]
 998 """
 999         self.tool.scm().blame = blame
1000
1001         test_port = self._setup_test_port()
1002
1003         old_builder_data = self.command.builder_data
1004
1005         def builder_data():
1006             old_builder_data()
1007             # have prototype-chocolate only fail on "MOCK Leopard".
1008             self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
1009     "tests": {
1010         "fast": {
1011             "dom": {
1012                 "prototype-taco.html": {
1013                     "expected": "PASS",
1014                     "actual": "PASS TEXT",
1015                     "is_unexpected": true
1016                 },
1017                 "prototype-chocolate.html": {
1018                     "expected": "FAIL",
1019                     "actual": "PASS"
1020                 },
1021                 "prototype-strawberry.html": {
1022                     "expected": "PASS",
1023                     "actual": "IMAGE PASS",
1024                     "is_unexpected": true
1025                 }
1026             }
1027         }
1028     }
1029 });""")
1030             return self.command._builder_data
1031
1032         self.command.builder_data = builder_data
1033
1034         self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
1035 crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ]
1036 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
1037 crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ]
1038 crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ]
1039 crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]
1040 crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]
1041 """)
1042
1043         self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
1044         self._write_test_file(test_port, 'fast/dom/prototype-strawberry.html', "Dummy test contents")
1045         self._write_test_file(test_port, 'fast/dom/prototype-chocolate.html', "Dummy test contents")
1046
1047         self.tool.executive = MockLineRemovingExecutive()
1048
1049         old_exact_matches = builders._exact_matches
1050         try:
1051             builders._exact_matches = {
1052                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
1053                 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
1054             }
1055
1056             self.command.tree_status = lambda: 'closed'
1057             self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool)
1058             self.assertEqual(self.tool.executive.calls, [])
1059
1060             self.command.tree_status = lambda: 'open'
1061             self.tool.executive.calls = []
1062             self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool)
1063
1064             self.assertEqual(self.tool.executive.calls, [
1065                 [
1066                     ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'],
1067                     ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'],
1068                     ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'],
1069                     ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'],
1070                 ],
1071                 [
1072                     ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'],
1073                     ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'],
1074                     ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'],
1075                     ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'],
1076                 ],
1077                 [
1078                     ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'fast/dom/prototype-chocolate.html'],
1079                     ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'png', 'fast/dom/prototype-strawberry.html'],
1080                     ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html'],
1081                 ],
1082                 ['git', 'cl', 'upload', '-f'],
1083                 ['git', 'pull'],
1084                 ['git', 'cl', 'dcommit', '-f'],
1085                 ['git', 'cl', 'set_close'],
1086             ])
1087
1088             # The mac ports should both be removed since they're the only ones in builders._exact_matches.
1089             self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """
1090 crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ]
1091 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
1092 crbug.com/24182 [ Linux Win ] fast/dom/prototype-chocolate.html [ NeedsRebaseline ]
1093 crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ]
1094 crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ]
1095 """)
1096         finally:
1097             builders._exact_matches = old_exact_matches
1098
1099     def test_execute_git_cl_hangs(self):
1100         def blame(path):
1101             return """
1102 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
1103 """
1104         self.tool.scm().blame = blame
1105
1106         test_port = self._setup_test_port()
1107
1108         old_builder_data = self.command.builder_data
1109
1110         def builder_data():
1111             old_builder_data()
1112             # have prototype-chocolate only fail on "MOCK Leopard".
1113             self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
1114     "tests": {
1115         "fast": {
1116             "dom": {
1117                 "prototype-taco.html": {
1118                     "expected": "PASS",
1119                     "actual": "PASS TEXT",
1120                     "is_unexpected": true
1121                 }
1122             }
1123         }
1124     }
1125 });""")
1126             return self.command._builder_data
1127
1128         self.command.builder_data = builder_data
1129
1130         self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
1131 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
1132 """)
1133
1134         self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
1135
1136         old_exact_matches = builders._exact_matches
1137         try:
1138             builders._exact_matches = {
1139                 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
1140             }
1141
1142             self.command.SECONDS_BEFORE_GIVING_UP = 0
1143             self.command.tree_status = lambda: 'open'
1144             self.tool.executive.calls = []
1145             self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool)
1146
1147             self.assertEqual(self.tool.executive.calls, [
1148                 [
1149                     ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'],
1150                 ],
1151                 [
1152                     ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'],
1153                 ],
1154                 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html']],
1155                 ['git', 'cl', 'upload', '-f'],
1156             ])
1157         finally:
1158             builders._exact_matches = old_exact_matches
1159
1160     def test_execute_test_passes_everywhere(self):
1161         def blame(path):
1162             return """
1163 6469e754a1 path/to/TestExpectations                   (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000   13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
1164 """
1165         self.tool.scm().blame = blame
1166
1167         test_port = self._setup_test_port()
1168
1169         old_builder_data = self.command.builder_data
1170
1171         def builder_data():
1172             self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({
1173     "tests": {
1174         "fast": {
1175             "dom": {
1176                 "prototype-taco.html": {
1177                     "expected": "FAIL",
1178                     "actual": "PASS",
1179                     "is_unexpected": true
1180                 }
1181             }
1182         }
1183     }
1184 });""")
1185             return self.command._builder_data
1186
1187         self.command.builder_data = builder_data
1188
1189         self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """
1190 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ]
1191 """)
1192
1193         self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents")
1194
1195         self.tool.executive = MockLineRemovingExecutive()
1196
1197         old_exact_matches = builders._exact_matches
1198         try:
1199             builders._exact_matches = {
1200                 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
1201                 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])},
1202             }
1203
1204             self.command.tree_status = lambda: 'open'
1205             self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool)
1206             self.assertEqual(self.tool.executive.calls, [
1207                 [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', '', 'fast/dom/prototype-taco.html']],
1208                 ['git', 'cl', 'upload', '-f'],
1209                 ['git', 'pull'],
1210                 ['git', 'cl', 'dcommit', '-f'],
1211                 ['git', 'cl', 'set_close'],
1212             ])
1213
1214             # The mac ports should both be removed since they're the only ones in builders._exact_matches.
1215             self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """
1216 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ]
1217 """)
1218         finally:
1219             builders._exact_matches = old_exact_matches
1220
1221
1222 class TestRebaselineOMatic(_BaseTestCase):
1223     command_constructor = RebaselineOMatic
1224
1225     def setUp(self):
1226         super(TestRebaselineOMatic, self).setUp()
1227         self._logs = []
1228
1229     def _mock_log_to_server(self, log=''):
1230         self._logs.append(log)
1231
1232     def test_run_logged_command(self):
1233         self.command._verbose = False
1234         self.command._post_log_to_server = self._mock_log_to_server
1235         self.command._run_logged_command(['echo', 'foo'])
1236         self.assertEqual(self.tool.executive.calls, [['echo', 'foo']])
1237         self.assertEqual(self._logs, ['MOCK STDOUT'])
1238
1239     def test_do_one_rebaseline(self):
1240         self.command._verbose = False
1241         self.command._post_log_to_server = self._mock_log_to_server
1242
1243         oc = OutputCapture()
1244         oc.capture_output()
1245         self.command._do_one_rebaseline()
1246         out, _, _ = oc.restore_output()
1247
1248         self.assertEqual(out, '')
1249         self.assertEqual(self.tool.executive.calls, [
1250             ['git', 'pull'],
1251             ['/mock-checkout/third_party/WebKit/Tools/Scripts/webkit-patch', 'auto-rebaseline'],
1252         ])
1253         self.assertEqual(self._logs, ['MOCK STDOUT'])
1254
1255     def test_do_one_rebaseline_verbose(self):
1256         self.command._verbose = True
1257         self.command._post_log_to_server = self._mock_log_to_server
1258
1259         oc = OutputCapture()
1260         oc.capture_output()
1261         self.command._do_one_rebaseline()
1262         out, _, _ = oc.restore_output()
1263
1264         self.assertEqual(out, 'MOCK STDOUT\n')
1265         self.assertEqual(self.tool.executive.calls, [
1266             ['git', 'pull'],
1267             ['/mock-checkout/third_party/WebKit/Tools/Scripts/webkit-patch', 'auto-rebaseline', '--verbose'],
1268         ])
1269         self.assertEqual(self._logs, ['MOCK STDOUT'])