From ef04aabf261f2f635f524e07de5840a3a3c03936 Mon Sep 17 00:00:00 2001 From: "ojan@chromium.org" Date: Wed, 14 Mar 2012 23:36:53 +0000 Subject: [PATCH] Have webkit-patch rebaseline-test update test_expectations.txt https://bugs.webkit.org/show_bug.cgi?id=81054 Reviewed by Dirk Pranke. This will allow us to use the All Failures tab in garden-o-matic to rebaseline tests without making test_expectations.txt stale. It's conservative and only removes lines if the test is not flaky and fails in a way the rebaseline would fix (e.g. CRASH/TIMEOUT will not be touched). * Scripts/webkitpy/layout_tests/models/test_expectations.py: (TestExpectations.remove_configuration_from_test): * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py: (RemoveConfigurationsTest): (RemoveConfigurationsTest.test_remove): (test_remove_line): * Scripts/webkitpy/tool/commands/rebaseline.py: (RebaselineTest._is_supported_port): (RebaselineTest): (RebaselineTest._update_expectations_file): (RebaselineTest._rebaseline_test_and_update_expectations): (RebaselineTest.execute): * Scripts/webkitpy/tool/commands/rebaseline_unittest.py: (TestRebaseline.test_rebaseline_updates_expectations_file): git-svn-id: http://svn.webkit.org/repository/webkit/trunk@110783 268f45cc-cd09-0410-ab3c-d52691b4dbfc --- Tools/ChangeLog | 29 +++++++++++++ .../layout_tests/models/test_expectations.py | 19 +++++++++ .../models/test_expectations_unittest.py | 47 ++++++++++++++++++++++ Tools/Scripts/webkitpy/tool/commands/rebaseline.py | 28 ++++++++++--- .../webkitpy/tool/commands/rebaseline_unittest.py | 15 +++++++ 5 files changed, 132 insertions(+), 6 deletions(-) diff --git a/Tools/ChangeLog b/Tools/ChangeLog index d19aae2..7f0a6b9 100644 --- a/Tools/ChangeLog +++ b/Tools/ChangeLog @@ -1,3 +1,32 @@ +2012-03-13 Ojan Vafai + + Have webkit-patch rebaseline-test update test_expectations.txt + https://bugs.webkit.org/show_bug.cgi?id=81054 + + Reviewed by Dirk Pranke. + + This will allow us to use the All Failures tab in garden-o-matic + to rebaseline tests without making test_expectations.txt stale. + + It's conservative and only removes lines if the test is + not flaky and fails in a way the rebaseline would fix (e.g. + CRASH/TIMEOUT will not be touched). + + * Scripts/webkitpy/layout_tests/models/test_expectations.py: + (TestExpectations.remove_configuration_from_test): + * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py: + (RemoveConfigurationsTest): + (RemoveConfigurationsTest.test_remove): + (test_remove_line): + * Scripts/webkitpy/tool/commands/rebaseline.py: + (RebaselineTest._is_supported_port): + (RebaselineTest): + (RebaselineTest._update_expectations_file): + (RebaselineTest._rebaseline_test_and_update_expectations): + (RebaselineTest.execute): + * Scripts/webkitpy/tool/commands/rebaseline_unittest.py: + (TestRebaseline.test_rebaseline_updates_expectations_file): + 2012-03-14 Jessie Berlin Crash in WKTR under addChromeInputField due to using 0 as a key in a HashMap diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py index f3ee6b5..7bc0695 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py @@ -816,6 +816,25 @@ class TestExpectations(object): def has_warnings(self): return self._has_warnings + def remove_configuration_from_test(self, test, test_configuration): + expectations_to_remove = [] + for expectation in self._expectations: + if expectation.name != test or expectation.is_flaky() or not expectation.parsed_expectations: + continue + if iter(expectation.parsed_expectations).next() not in (FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO): + continue + if test_configuration not in expectation.matching_configurations: + continue + + expectation.matching_configurations.remove(test_configuration) + if not expectation.matching_configurations: + expectations_to_remove.append(expectation) + + for expectation in expectations_to_remove: + self._expectations.remove(expectation) + + return TestExpectationSerializer.list_to_string(self._expectations, self._parser._test_configuration_converter) + def remove_rebaselined_tests(self, except_these_tests): """Returns a copy of the expectations with the tests removed.""" def without_rebaseline_modifier(expectation): diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py index 6110034..ce5c932 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py @@ -378,6 +378,53 @@ BUGX : failures/expected/text.html = TEXT "BUG_TEST XP : passes/text.html = TEXT\n") +class RemoveConfigurationsTest(Base): + def test_remove(self): + host = MockHost() + test_port = host.port_factory.get('test-win-xp', None) + test_port.test_exists = lambda test: True + test_port.test_isfile = lambda test: True + + test_config = test_port.test_configuration() + expectations = TestExpectations(test_port, + tests=self.get_basic_tests(), + expectations="""BUGX LINUX WIN RELEASE : failures/expected/foo.html = TEXT +BUGY WIN DEBUG : failures/expected/foo.html = CRASH +""", + test_config=test_config, + is_lint_mode=False, + overrides=None) + + actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) + + self.assertEqual("""BUGX LINUX VISTA WIN7 RELEASE : failures/expected/foo.html = TEXT +BUGY WIN DEBUG : failures/expected/foo.html = CRASH +""", actual_expectations) + + def test_remove_line(self): + host = MockHost() + test_port = host.port_factory.get('test-win-xp', None) + test_port.test_exists = lambda test: True + test_port.test_isfile = lambda test: True + + test_config = test_port.test_configuration() + expectations = TestExpectations(test_port, + tests=None, + expectations="""BUGX WIN RELEASE : failures/expected/foo.html = TEXT +BUGY WIN DEBUG : failures/expected/foo.html = CRASH +""", + test_config=test_config, + is_lint_mode=False, + overrides=None) + + actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) + actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration()) + actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) + + self.assertEqual("""BUGY WIN DEBUG : failures/expected/foo.html = CRASH +""", actual_expectations) + + class RebaseliningTest(Base): """Test rebaselining-specific functionality.""" def assertRemove(self, input_expectations, tests, expected_expectations): diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py index c4a206d..3897795 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py @@ -40,6 +40,7 @@ from webkitpy.common.system.executive import ScriptError from webkitpy.common.system.user import User from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter from webkitpy.layout_tests.models import test_failures +from webkitpy.layout_tests.models.test_configuration import TestConfiguration from webkitpy.layout_tests.models.test_expectations import TestExpectations from webkitpy.layout_tests.port import builders from webkitpy.tool.grammar import pluralize @@ -108,6 +109,17 @@ class RebaselineTest(AbstractDeclarativeCommand): if not self._tool.scm().exists(target_baseline): self._tool.scm().add(target_baseline) + def _update_expectations_file(self, builder_name, test_name): + port = self._tool.port_factory.get_from_builder_name(builder_name) + expectationsString = port.test_expectations() + expectations = TestExpectations(port, None, expectationsString, port.test_configuration()) + + for test_configuration in port.all_test_configurations(): + if test_configuration.version == port.test_configuration().version: + expectationsString = expectations.remove_configuration_from_test(test_name, test_configuration) + + self._tool.filesystem.write_text_file(port.path_to_test_expectations_file(), expectationsString) + def _test_root(self, test_name): return os.path.splitext(test_name)[0] @@ -130,13 +142,17 @@ class RebaselineTest(AbstractDeclarativeCommand): print "Retrieving %s." % source_baseline self._save_baseline(self._tool.web.get_binary(source_baseline, convert_404_to_None=True), target_baseline) - def execute(self, options, args, tool): + def _rebaseline_test_and_update_expectations(self, builder_name, test_name, platforms_to_move_existing_baselines_to): for suffix in _baseline_suffix_list: - if len(args) > 2: - platforms_to_move_existing_baselines_to = args[2:] - else: - platforms_to_move_existing_baselines_to = None - self._rebaseline_test(args[0], args[1], platforms_to_move_existing_baselines_to, suffix) + self._rebaseline_test(builder_name, test_name, platforms_to_move_existing_baselines_to, suffix) + self._update_expectations_file(builder_name, test_name) + + def execute(self, options, args, tool): + if len(args) > 2: + platforms_to_move_existing_baselines_to = args[2:] + else: + platforms_to_move_existing_baselines_to = None + self._rebaseline_test_and_update_expectations(args[0], args[1], platforms_to_move_existing_baselines_to) class OptimizeBaselines(AbstractDeclarativeCommand): diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py index a817fa0..47d0961 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py @@ -42,6 +42,21 @@ class TestRebaseline(unittest.TestCase): build = Mock() OutputCapture().assert_outputs(self, command._tests_to_update, [build]) + def test_rebaseline_updates_expectations_file(self): + command = RebaselineTest() + tool = MockTool() + command.bind_to_tool(tool) + + lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") + tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), "BUGX MAC : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html\n") + tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents") + + expected_stdout = "Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png.\nRetrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt.\n" + OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_stdout=expected_stdout) + + new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) + self.assertEqual(new_expectations, "BUGX SNOWLEOPARD : userscripts/another-test.html = IMAGE\nBUGX LEOPARD : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html\n") + def test_rebaseline_test(self): command = RebaselineTest() command.bind_to_tool(MockTool()) -- 2.7.4