src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import webkitpy.thirdparty.unittest2 as unittest
  30
  31 from webkitpy.common.host_mock import MockHost
  32 from webkitpy.common.system.outputcapture import OutputCapture
  33
  34 from webkitpy.layout_tests.models.test_configuration import *
  35 from webkitpy.layout_tests.models.test_expectations import *
  36
  37 try:
  38     from collections import OrderedDict
  39 except ImportError:
  40     # Needed for Python < 2.7
  41     from webkitpy.thirdparty.ordered_dict import OrderedDict
  42
  43
  44 class Base(unittest.TestCase):
  45     # Note that all of these tests are written assuming the configuration
  46     # being tested is Windows XP, Release build.
  47
  48     def __init__(self, testFunc):
  49         host = MockHost()
  50         self._port = host.port_factory.get('test-win-xp', None)
  51         self._exp = None
  52         unittest.TestCase.__init__(self, testFunc)
  53
  54     def get_test(self, test_name):
  55         # FIXME: Remove this routine and just reference test names directly.
  56         return test_name
  57
  58     def get_basic_tests(self):
  59         return [self.get_test('failures/expected/text.html'),
  60                 self.get_test('failures/expected/image_checksum.html'),
  61                 self.get_test('failures/expected/crash.html'),
  62                 self.get_test('failures/expected/needsrebaseline.html'),
  63                 self.get_test('failures/expected/needsmanualrebaseline.html'),
  64                 self.get_test('failures/expected/missing_text.html'),
  65                 self.get_test('failures/expected/image.html'),
  66                 self.get_test('passes/text.html')]
  67
  68     def get_basic_expectations(self):
  69         return """
  70 Bug(test) failures/expected/text.html [ Failure ]
  71 Bug(test) failures/expected/crash.html [ WontFix ]
  72 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ]
  73 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
  74 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
  75 Bug(test) failures/expected/image_checksum.html [ WontFix ]
  76 Bug(test) failures/expected/image.html [ WontFix Mac ]
  77 """
  78
  79     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
  80         expectations_dict = OrderedDict()
  81         expectations_dict['expectations'] = expectations
  82         if overrides:
  83             expectations_dict['overrides'] = overrides
  84         self._port.expectations_dict = lambda: expectations_dict
  85         expectations_to_lint = expectations_dict if is_lint_mode else None
  86         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_dict=expectations_to_lint, is_lint_mode=is_lint_mode)
  87
  88     def assert_exp_list(self, test, results):
  89         self.assertEqual(self._exp.get_expectations(self.get_test(test)), set(results))
  90
  91     def assert_exp(self, test, result):
  92         self.assert_exp_list(test, [result])
  93
  94     def assert_bad_expectations(self, expectations, overrides=None):
  95         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
  96
  97
  98 class BasicTests(Base):
  99     def test_basic(self):
 100         self.parse_exp(self.get_basic_expectations())
 101         self.assert_exp('failures/expected/text.html', FAIL)
 102         self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX, SKIP])
 103         self.assert_exp('passes/text.html', PASS)
 104         self.assert_exp('failures/expected/image.html', PASS)
 105
 106
 107 class MiscTests(Base):
 108     def test_multiple_results(self):
 109         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
 110         self.assertEqual(self._exp.get_expectations(
 111             self.get_test('failures/expected/text.html')),
 112             set([FAIL, CRASH]))
 113
 114     def test_result_was_expected(self):
 115         # test basics
 116         self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False), True)
 117         self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False), False)
 118
 119         # test handling of SKIPped tests and results
 120         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False), True)
 121         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([LEAK]), test_needs_rebaselining=False), True)
 122
 123         # test handling of MISSING results and the REBASELINE specifier
 124         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True), True)
 125         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False), False)
 126
 127         self.assertTrue(TestExpectations.result_was_expected(PASS, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 128         self.assertTrue(TestExpectations.result_was_expected(MISSING, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 129         self.assertTrue(TestExpectations.result_was_expected(TEXT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 130         self.assertTrue(TestExpectations.result_was_expected(IMAGE, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 131         self.assertTrue(TestExpectations.result_was_expected(IMAGE_PLUS_TEXT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 132         self.assertTrue(TestExpectations.result_was_expected(AUDIO, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 133         self.assertFalse(TestExpectations.result_was_expected(TIMEOUT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 134         self.assertFalse(TestExpectations.result_was_expected(CRASH, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 135         self.assertFalse(TestExpectations.result_was_expected(LEAK, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 136
 137     def test_remove_pixel_failures(self):
 138         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
 139         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
 140         self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
 141         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
 142         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
 143
 144     def test_suffixes_for_expectations(self):
 145         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
 146         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
 147         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
 148         self.assertEqual(TestExpectations.suffixes_for_expectations(set()), set())
 149
 150     def test_category_expectations(self):
 151         # This test checks unknown tests are not present in the
 152         # expectations and that known test part of a test category is
 153         # present in the expectations.
 154         exp_str = 'Bug(x) failures/expected [ WontFix ]'
 155         self.parse_exp(exp_str)
 156         test_name = 'failures/expected/unknown-test.html'
 157         unknown_test = self.get_test(test_name)
 158         self.assertRaises(KeyError, self._exp.get_expectations,
 159                           unknown_test)
 160         self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
 161
 162     def test_get_expectations_string(self):
 163         self.parse_exp(self.get_basic_expectations())
 164         self.assertEqual(self._exp.get_expectations_string(
 165                           self.get_test('failures/expected/text.html')),
 166                           'FAIL')
 167
 168     def test_expectation_to_string(self):
 169         # Normal cases are handled by other tests.
 170         self.parse_exp(self.get_basic_expectations())
 171         self.assertRaises(ValueError, self._exp.expectation_to_string,
 172                           -1)
 173
 174     def test_get_test_set(self):
 175         # Handle some corner cases for this routine not covered by other tests.
 176         self.parse_exp(self.get_basic_expectations())
 177         s = self._exp.get_test_set(WONTFIX)
 178         self.assertEqual(s,
 179             set([self.get_test('failures/expected/crash.html'),
 180                  self.get_test('failures/expected/image_checksum.html')]))
 181
 182     def test_needs_rebaseline_reftest(self):
 183         try:
 184             filesystem = self._port.host.filesystem
 185             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline.html'), 'content')
 186             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline-expected.html'), 'content')
 187             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline.html'), 'content')
 188             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline-expected.html'), 'content')
 189             self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [ NeedsRebaseline ]
 190 Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]""", is_lint_mode=True)
 191             self.assertFalse(True, "ParseError wasn't raised")
 192         except ParseError, e:
 193             warnings = """expectations:1 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline failures/expected/needsrebaseline.html
 194 expectations:2 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline failures/expected/needsmanualrebaseline.html"""
 195             self.assertEqual(str(e), warnings)
 196
 197     def test_parse_warning(self):
 198         try:
 199             filesystem = self._port.host.filesystem
 200             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
 201             self.get_test('disabled-test.html-disabled'),
 202             self.parse_exp("Bug(user) [ FOO ] failures/expected/text.html [ Failure ]\n"
 203                 "Bug(user) non-existent-test.html [ Failure ]\n"
 204                 "Bug(user) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
 205             self.assertFalse(True, "ParseError wasn't raised")
 206         except ParseError, e:
 207             warnings = ("expectations:1 Unrecognized specifier 'foo' failures/expected/text.html\n"
 208                         "expectations:2 Path does not exist. non-existent-test.html")
 209             self.assertEqual(str(e), warnings)
 210
 211     def test_parse_warnings_are_logged_if_not_in_lint_mode(self):
 212         oc = OutputCapture()
 213         try:
 214             oc.capture_output()
 215             self.parse_exp('-- this should be a syntax error', is_lint_mode=False)
 216         finally:
 217             _, _, logs = oc.restore_output()
 218             self.assertNotEquals(logs, '')
 219
 220     def test_error_on_different_platform(self):
 221         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
 222         self.assertRaises(ParseError, self.parse_exp,
 223             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
 224             is_lint_mode=True)
 225
 226     def test_error_on_different_build_type(self):
 227         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
 228         self.assertRaises(ParseError, self.parse_exp,
 229             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
 230             is_lint_mode=True)
 231
 232     def test_overrides(self):
 233         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
 234                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
 235         self.assert_exp_list('failures/expected/text.html', [FAIL, IMAGE])
 236
 237     def test_overrides__directory(self):
 238         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
 239                        "Bug(override) failures/expected [ Crash ]")
 240         self.assert_exp_list('failures/expected/text.html', [FAIL, CRASH])
 241         self.assert_exp_list('failures/expected/image.html', [CRASH])
 242
 243     def test_overrides__duplicate(self):
 244         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
 245                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
 246                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
 247
 248     def test_pixel_tests_flag(self):
 249         def match(test, result, pixel_tests_enabled):
 250             return self._exp.matches_an_expected_result(
 251                 self.get_test(test), result, pixel_tests_enabled)
 252
 253         self.parse_exp(self.get_basic_expectations())
 254         self.assertTrue(match('failures/expected/text.html', FAIL, True))
 255         self.assertTrue(match('failures/expected/text.html', FAIL, False))
 256         self.assertFalse(match('failures/expected/text.html', CRASH, True))
 257         self.assertFalse(match('failures/expected/text.html', CRASH, False))
 258         self.assertTrue(match('failures/expected/image_checksum.html', PASS, True))
 259         self.assertTrue(match('failures/expected/image_checksum.html', PASS, False))
 260         self.assertTrue(match('failures/expected/crash.html', PASS, False))
 261         self.assertTrue(match('failures/expected/needsrebaseline.html', TEXT, True))
 262         self.assertFalse(match('failures/expected/needsrebaseline.html', CRASH, True))
 263         self.assertTrue(match('failures/expected/needsmanualrebaseline.html', TEXT, True))
 264         self.assertFalse(match('failures/expected/needsmanualrebaseline.html', CRASH, True))
 265         self.assertTrue(match('passes/text.html', PASS, False))
 266
 267     def test_more_specific_override_resets_skip(self):
 268         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
 269                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
 270         self.assert_exp('failures/expected/text.html', IMAGE)
 271         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
 272                                                      'failures/expected/text.html') in
 273                          self._exp.get_tests_with_result_type(SKIP))
 274
 275     def test_bot_test_expectations(self):
 276         """Test that expectations are merged rather than overridden when using flaky option 'unexpected'."""
 277         test_name1 = 'failures/expected/text.html'
 278         test_name2 = 'passes/text.html'
 279
 280         expectations_dict = OrderedDict()
 281         expectations_dict['expectations'] = "Bug(x) %s [ ImageOnlyFailure ]\nBug(x) %s [ Slow ]\n" % (test_name1, test_name2)
 282         self._port.expectations_dict = lambda: expectations_dict
 283
 284         expectations = TestExpectations(self._port, self.get_basic_tests())
 285         self.assertEqual(expectations.get_expectations(self.get_test(test_name1)), set([IMAGE]))
 286         self.assertEqual(expectations.get_expectations(self.get_test(test_name2)), set([SLOW]))
 287
 288         def bot_expectations():
 289             return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']}
 290         self._port.bot_expectations = bot_expectations
 291         self._port._options.ignore_flaky_tests = 'unexpected'
 292
 293         expectations = TestExpectations(self._port, self.get_basic_tests())
 294         self.assertEqual(expectations.get_expectations(self.get_test(test_name1)), set([PASS, IMAGE, TIMEOUT]))
 295         self.assertEqual(expectations.get_expectations(self.get_test(test_name2)), set([CRASH, SLOW]))
 296
 297 class SkippedTests(Base):
 298     def check(self, expectations, overrides, skips, lint=False, expected_results=[WONTFIX, SKIP, FAIL]):
 299         port = MockHost().port_factory.get('test-win-xp')
 300         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
 301         expectations_dict = OrderedDict()
 302         expectations_dict['expectations'] = expectations
 303         if overrides:
 304             expectations_dict['overrides'] = overrides
 305         port.expectations_dict = lambda: expectations_dict
 306         port.skipped_layout_tests = lambda tests: set(skips)
 307         expectations_to_lint = expectations_dict if lint else None
 308         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_dict=expectations_to_lint, is_lint_mode=lint)
 309         self.assertEqual(exp.get_expectations('failures/expected/text.html'), set(expected_results))
 310
 311     def test_skipped_tests_work(self):
 312         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'], expected_results=[WONTFIX, SKIP])
 313
 314     def test_duplicate_skipped_test_fails_lint(self):
 315         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n',
 316             overrides=None, skips=['failures/expected/text.html'], lint=True)
 317
 318     def test_skipped_file_overrides_expectations(self):
 319         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
 320                    skips=['failures/expected/text.html'])
 321
 322     def test_skipped_dir_overrides_expectations(self):
 323         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
 324                    skips=['failures/expected'])
 325
 326     def test_skipped_file_overrides_overrides(self):
 327         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
 328                    skips=['failures/expected/text.html'])
 329
 330     def test_skipped_dir_overrides_overrides(self):
 331         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
 332                    skips=['failures/expected'])
 333
 334     def test_skipped_entry_dont_exist(self):
 335         port = MockHost().port_factory.get('test-win-xp')
 336         expectations_dict = OrderedDict()
 337         expectations_dict['expectations'] = ''
 338         port.expectations_dict = lambda: expectations_dict
 339         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
 340         capture = OutputCapture()
 341         capture.capture_output()
 342         exp = TestExpectations(port)
 343         _, _, logs = capture.restore_output()
 344         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
 345
 346     def test_expectations_string(self):
 347         self.parse_exp(self.get_basic_expectations())
 348         notrun = 'failures/expected/text.html'
 349         self._exp.add_extra_skipped_tests([notrun])
 350         self.assertEqual('NOTRUN', self._exp.get_expectations_string(notrun))
 351
 352
 353 class ExpectationSyntaxTests(Base):
 354     def test_unrecognized_expectation(self):
 355         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
 356
 357     def test_macro(self):
 358         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
 359         self.parse_exp(exp_str)
 360         self.assert_exp('failures/expected/text.html', FAIL)
 361
 362     def assert_tokenize_exp(self, line, bugs=None, specifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
 363         bugs = bugs or []
 364         specifiers = specifiers or []
 365         expectations = expectations or []
 366         warnings = warnings or []
 367         filename = 'TestExpectations'
 368         line_number = '1'
 369         expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number)
 370         self.assertEqual(expectation_line.warnings, warnings)
 371         self.assertEqual(expectation_line.name, name)
 372         self.assertEqual(expectation_line.filename, filename)
 373         self.assertEqual(expectation_line.line_numbers, line_number)
 374         if not warnings:
 375             self.assertEqual(expectation_line.specifiers, specifiers)
 376             self.assertEqual(expectation_line.expectations, expectations)
 377
 378     def test_comments(self):
 379         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
 380         self.assert_tokenize_exp("foo.html [ Pass ] # comment", comment="# comment", expectations=['PASS'], specifiers=[])
 381
 382     def test_config_specifiers(self):
 383         self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', specifiers=['MAC'], expectations=['FAIL'])
 384
 385     def test_unknown_config(self):
 386         self.assert_tokenize_exp('[ Foo ] foo.html [ Pass ]', specifiers=['Foo'], expectations=['PASS'])
 387
 388     def test_unknown_expectation(self):
 389         self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"'])
 390
 391     def test_skip(self):
 392         self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectations=['SKIP'])
 393
 394     def test_slow(self):
 395         self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectations=['SLOW'])
 396
 397     def test_wontfix(self):
 398         self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectations=['WONTFIX', 'SKIP'])
 399         self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', specifiers=[], expectations=['WONTFIX', 'SKIP'],
 400             warnings=['A test marked Skip or WontFix must not have other expectations.'])
 401
 402     def test_blank_line(self):
 403         self.assert_tokenize_exp('', name=None)
 404
 405     def test_warnings(self):
 406         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.', 'Missing expectations.'], name=None)
 407         self.assert_tokenize_exp('[ [', warnings=['unexpected "["', 'Missing expectations.'], name=None)
 408         self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"', 'Missing expectations.'], name=None)
 409
 410         self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.', 'Missing expectations.'])
 411         self.assert_tokenize_exp('foo.html', warnings=['Missing expectations.'])
 412
 413
 414 class SemanticTests(Base):
 415     def test_bug_format(self):
 416         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
 417
 418     def test_bad_bugid(self):
 419         try:
 420             self.parse_exp('crbug/1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
 421             self.fail('should have raised an error about a bad bug identifier')
 422         except ParseError, exp:
 423             self.assertEqual(len(exp.warnings), 3)
 424
 425     def test_missing_bugid(self):
 426         self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=False)
 427         self.assertFalse(self._exp.has_warnings())
 428
 429         try:
 430             self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=True)
 431         except ParseError, exp:
 432             self.assertEqual(exp.warnings, ['expectations:1 Test lacks BUG specifier. failures/expected/text.html'])
 433
 434     def test_skip_and_wontfix(self):
 435         # Skip is not allowed to have other expectations as well, because those
 436         # expectations won't be exercised and may become stale .
 437         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
 438         self.assertTrue(self._exp.has_warnings())
 439
 440         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
 441         self.assertTrue(self._exp.has_warnings())
 442
 443         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
 444         self.assertTrue(self._exp.has_warnings())
 445
 446     def test_rebaseline(self):
 447         # Can't lint a file w/ 'REBASELINE' in it.
 448         self.assertRaises(ParseError, self.parse_exp,
 449             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
 450             is_lint_mode=True)
 451
 452     def test_duplicates(self):
 453         self.assertRaises(ParseError, self.parse_exp, """
 454 Bug(exp) failures/expected/text.html [ Failure ]
 455 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
 456
 457         self.assertRaises(ParseError, self.parse_exp,
 458             self.get_basic_expectations(), overrides="""
 459 Bug(override) failures/expected/text.html [ Failure ]
 460 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
 461
 462     def test_duplicate_with_line_before_preceding_line(self):
 463         self.assert_bad_expectations("""Bug(exp) [ Debug ] failures/expected/text.html [ Failure ]
 464 Bug(exp) [ Release ] failures/expected/text.html [ Failure ]
 465 Bug(exp) [ Debug ] failures/expected/text.html [ Failure ]
 466 """)
 467
 468     def test_missing_file(self):
 469         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
 470         self.assertTrue(self._exp.has_warnings(), 1)
 471
 472
 473 class PrecedenceTests(Base):
 474     def test_file_over_directory(self):
 475         # This tests handling precedence of specific lines over directories
 476         # and tests expectations covering entire directories.
 477         exp_str = """
 478 Bug(x) failures/expected/text.html [ Failure ]
 479 Bug(y) failures/expected [ WontFix ]
 480 """
 481         self.parse_exp(exp_str)
 482         self.assert_exp('failures/expected/text.html', FAIL)
 483         self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
 484
 485         exp_str = """
 486 Bug(x) failures/expected [ WontFix ]
 487 Bug(y) failures/expected/text.html [ Failure ]
 488 """
 489         self.parse_exp(exp_str)
 490         self.assert_exp('failures/expected/text.html', FAIL)
 491         self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
 492
 493     def test_ambiguous(self):
 494         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
 495                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
 496
 497     def test_more_specifiers(self):
 498         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
 499                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
 500
 501     def test_order_in_file(self):
 502         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
 503                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
 504
 505     def test_macro_overrides(self):
 506         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
 507                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
 508
 509
 510 class RemoveConfigurationsTest(Base):
 511     def test_remove(self):
 512         host = MockHost()
 513         test_port = host.port_factory.get('test-win-xp', None)
 514         test_port.test_exists = lambda test: True
 515         test_port.test_isfile = lambda test: True
 516
 517         test_config = test_port.test_configuration()
 518         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
 519 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 520 """}
 521         expectations = TestExpectations(test_port, self.get_basic_tests())
 522
 523         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 524
 525         self.assertEqual("""Bug(x) [ Linux Win7 Release ] failures/expected/foo.html [ Failure ]
 526 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 527 """, actual_expectations)
 528
 529     def test_remove_needs_rebaseline(self):
 530         host = MockHost()
 531         test_port = host.port_factory.get('test-win-xp', None)
 532         test_port.test_exists = lambda test: True
 533         test_port.test_isfile = lambda test: True
 534
 535         test_config = test_port.test_configuration()
 536         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Win ] failures/expected/foo.html [ NeedsRebaseline ]
 537 """}
 538         expectations = TestExpectations(test_port, self.get_basic_tests())
 539
 540         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 541
 542         self.assertEqual("""Bug(x) [ XP Debug ] failures/expected/foo.html [ NeedsRebaseline ]
 543 Bug(x) [ Win7 ] failures/expected/foo.html [ NeedsRebaseline ]
 544 """, actual_expectations)
 545
 546     def test_remove_line_with_comments(self):
 547         host = MockHost()
 548         test_port = host.port_factory.get('test-win-xp', None)
 549         test_port.test_exists = lambda test: True
 550         test_port.test_isfile = lambda test: True
 551
 552         test_config = test_port.test_configuration()
 553         test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 554
 555  # This comment line should get stripped. As should the preceding line.
 556 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 557 """}
 558         expectations = TestExpectations(test_port)
 559
 560         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 561         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
 562
 563         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 564 """, actual_expectations)
 565
 566     def test_remove_line_with_comments_at_start(self):
 567         host = MockHost()
 568         test_port = host.port_factory.get('test-win-xp', None)
 569         test_port.test_exists = lambda test: True
 570         test_port.test_isfile = lambda test: True
 571
 572         test_config = test_port.test_configuration()
 573         test_port.expectations_dict = lambda: {'expectations': """
 574  # This comment line should get stripped. As should the preceding line.
 575 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 576
 577 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 578 """}
 579         expectations = TestExpectations(test_port)
 580
 581         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 582         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
 583
 584         self.assertEqual("""
 585 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 586 """, actual_expectations)
 587
 588     def test_remove_line_with_comments_at_end_with_no_trailing_newline(self):
 589         host = MockHost()
 590         test_port = host.port_factory.get('test-win-xp', None)
 591         test_port.test_exists = lambda test: True
 592         test_port.test_isfile = lambda test: True
 593
 594         test_config = test_port.test_configuration()
 595         test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 596
 597  # This comment line should get stripped. As should the preceding line.
 598 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]"""}
 599         expectations = TestExpectations(test_port)
 600
 601         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 602         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
 603
 604         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]""", actual_expectations)
 605
 606     def test_remove_line_leaves_comments_for_next_line(self):
 607         host = MockHost()
 608         test_port = host.port_factory.get('test-win-xp', None)
 609         test_port.test_exists = lambda test: True
 610         test_port.test_isfile = lambda test: True
 611
 612         test_config = test_port.test_configuration()
 613         test_port.expectations_dict = lambda: {'expectations': """
 614  # This comment line should not get stripped.
 615 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 616 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 617 """}
 618         expectations = TestExpectations(test_port)
 619
 620         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 621         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
 622
 623         self.assertEqual("""
 624  # This comment line should not get stripped.
 625 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 626 """, actual_expectations)
 627
 628     def test_remove_line_no_whitespace_lines(self):
 629         host = MockHost()
 630         test_port = host.port_factory.get('test-win-xp', None)
 631         test_port.test_exists = lambda test: True
 632         test_port.test_isfile = lambda test: True
 633
 634         test_config = test_port.test_configuration()
 635         test_port.expectations_dict = lambda: {'expectations': """
 636  # This comment line should get stripped.
 637 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 638  # This comment line should not get stripped.
 639 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 640 """}
 641         expectations = TestExpectations(test_port)
 642
 643         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 644         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
 645
 646         self.assertEqual(""" # This comment line should not get stripped.
 647 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 648 """, actual_expectations)
 649
 650     def test_remove_first_line(self):
 651         host = MockHost()
 652         test_port = host.port_factory.get('test-win-xp', None)
 653         test_port.test_exists = lambda test: True
 654         test_port.test_isfile = lambda test: True
 655
 656         test_config = test_port.test_configuration()
 657         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 658  # This comment line should not get stripped.
 659 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 660 """}
 661         expectations = TestExpectations(test_port)
 662
 663         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 664         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
 665
 666         self.assertEqual(""" # This comment line should not get stripped.
 667 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 668 """, actual_expectations)
 669
 670     def test_remove_flaky_line(self):
 671         host = MockHost()
 672         test_port = host.port_factory.get('test-win-xp', None)
 673         test_port.test_exists = lambda test: True
 674         test_port.test_isfile = lambda test: True
 675
 676         test_config = test_port.test_configuration()
 677         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ] failures/expected/foo.html [ Failure Timeout ]
 678 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ]
 679 """}
 680         expectations = TestExpectations(test_port)
 681
 682         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
 683         actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
 684
 685         self.assertEqual("""Bug(x) [ Win Debug ] failures/expected/foo.html [ Failure Timeout ]
 686 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ]
 687 """, actual_expectations)
 688
 689
 690 class RebaseliningTest(Base):
 691     def test_get_rebaselining_failures(self):
 692         # Make sure we find a test as needing a rebaseline even if it is not marked as a failure.
 693         self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n')
 694         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
 695
 696         self.parse_exp(self.get_basic_expectations())
 697         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
 698
 699
 700 class TestExpectationsParserTests(unittest.TestCase):
 701     def __init__(self, testFunc):
 702         host = MockHost()
 703         test_port = host.port_factory.get('test-win-xp', None)
 704         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
 705         unittest.TestCase.__init__(self, testFunc)
 706         self._parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], is_lint_mode=False)
 707
 708     def test_expectation_line_for_test(self):
 709         # This is kind of a silly test, but it at least ensures that we don't throw an error.
 710         test_name = 'foo/test.html'
 711         expectations = set(["PASS", "IMAGE"])
 712
 713         expectation_line = TestExpectationLine()
 714         expectation_line.original_string = test_name
 715         expectation_line.name = test_name
 716         expectation_line.filename = '<Bot TestExpectations>'
 717         expectation_line.line_numbers = '0'
 718         expectation_line.expectations = expectations
 719         self._parser._parse_line(expectation_line)
 720
 721         self.assertEqual(self._parser.expectation_line_for_test(test_name, expectations), expectation_line)
 722
 723
 724 class TestExpectationSerializationTests(unittest.TestCase):
 725     def __init__(self, testFunc):
 726         host = MockHost()
 727         test_port = host.port_factory.get('test-win-xp', None)
 728         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
 729         unittest.TestCase.__init__(self, testFunc)
 730
 731     def _tokenize(self, line):
 732         return TestExpectationParser._tokenize_line('path', line, 0)
 733
 734     def assert_round_trip(self, in_string, expected_string=None):
 735         expectation = self._tokenize(in_string)
 736         if expected_string is None:
 737             expected_string = in_string
 738         self.assertEqual(expected_string, expectation.to_string(self._converter))
 739
 740     def assert_list_round_trip(self, in_string, expected_string=None):
 741         host = MockHost()
 742         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], is_lint_mode=False)
 743         expectations = parser.parse('path', in_string)
 744         if expected_string is None:
 745             expected_string = in_string
 746         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
 747
 748     def test_unparsed_to_string(self):
 749         expectation = TestExpectationLine()
 750
 751         self.assertEqual(expectation.to_string(self._converter), '')
 752         expectation.comment = ' Qux.'
 753         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
 754         expectation.name = 'bar'
 755         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
 756         expectation.specifiers = ['foo']
 757         # FIXME: case should be preserved here but we can't until we drop the old syntax.
 758         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
 759         expectation.expectations = ['bAz']
 760         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
 761         expectation.expectations = ['bAz1', 'baZ2']
 762         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
 763         expectation.specifiers = ['foo1', 'foO2']
 764         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
 765         expectation.warnings.append('Oh the horror.')
 766         self.assertEqual(expectation.to_string(self._converter), '')
 767         expectation.original_string = 'Yes it is!'
 768         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
 769
 770     def test_unparsed_list_to_string(self):
 771         expectation = TestExpectationLine()
 772         expectation.comment = 'Qux.'
 773         expectation.name = 'bar'
 774         expectation.specifiers = ['foo']
 775         expectation.expectations = ['bAz1', 'baZ2']
 776         # FIXME: case should be preserved here but we can't until we drop the old syntax.
 777         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
 778
 779     def test_parsed_to_string(self):
 780         expectation_line = TestExpectationLine()
 781         expectation_line.bugs = ['Bug(x)']
 782         expectation_line.name = 'test/name/for/realz.html'
 783         expectation_line.parsed_expectations = set([IMAGE])
 784         self.assertEqual(expectation_line.to_string(self._converter), None)
 785         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
 786         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
 787         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
 788         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
 789
 790     def test_serialize_parsed_expectations(self):
 791         expectation_line = TestExpectationLine()
 792         expectation_line.parsed_expectations = set([])
 793         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
 794         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
 795         expectation_line.parsed_expectations = set([FAIL])
 796         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
 797         expectation_line.parsed_expectations = set([PASS, IMAGE])
 798         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'image pass')
 799         expectation_line.parsed_expectations = set([FAIL, PASS])
 800         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
 801
 802     def test_serialize_parsed_specifier_string(self):
 803         expectation_line = TestExpectationLine()
 804         expectation_line.bugs = ['garden-o-matic']
 805         expectation_line.parsed_specifiers = ['the', 'for']
 806         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, []), 'for the')
 807         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, ['win']), 'for the win')
 808         expectation_line.bugs = []
 809         expectation_line.parsed_specifiers = []
 810         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, []), '')
 811         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, ['win']), 'win')
 812
 813     def test_format_line(self):
 814         self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
 815         self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
 816
 817     def test_string_roundtrip(self):
 818         self.assert_round_trip('')
 819         self.assert_round_trip('[')
 820         self.assert_round_trip('FOO [')
 821         self.assert_round_trip('FOO ] bar')
 822         self.assert_round_trip('  FOO [')
 823         self.assert_round_trip('    [ FOO ] ')
 824         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
 825         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
 826         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
 827         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
 828         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
 829         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
 830         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
 831         self.assert_round_trip('FOO ] ] bar ==== BAZ')
 832         self.assert_round_trip('=')
 833         self.assert_round_trip('#')
 834         self.assert_round_trip('# ')
 835         self.assert_round_trip('# Foo')
 836         self.assert_round_trip('# Foo')
 837         self.assert_round_trip('# Foo :')
 838         self.assert_round_trip('# Foo : =')
 839
 840     def test_list_roundtrip(self):
 841         self.assert_list_round_trip('')
 842         self.assert_list_round_trip('\n')
 843         self.assert_list_round_trip('\n\n')
 844         self.assert_list_round_trip('bar')
 845         self.assert_list_round_trip('bar\n# Qux.')
 846         self.assert_list_round_trip('bar\n# Qux.\n')
 847
 848     def test_reconstitute_only_these(self):
 849         lines = []
 850         reconstitute_only_these = []
 851
 852         def add_line(matching_configurations, reconstitute):
 853             expectation_line = TestExpectationLine()
 854             expectation_line.original_string = "Nay"
 855             expectation_line.bugs = ['Bug(x)']
 856             expectation_line.name = 'Yay'
 857             expectation_line.parsed_expectations = set([IMAGE])
 858             expectation_line.matching_configurations = matching_configurations
 859             lines.append(expectation_line)
 860             if reconstitute:
 861                 reconstitute_only_these.append(expectation_line)
 862
 863         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
 864         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
 865         serialized = TestExpectations.list_to_string(lines, self._converter)
 866         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
 867         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
 868         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
 869
 870     def disabled_test_string_whitespace_stripping(self):
 871         # FIXME: Re-enable this test once we rework the code to no longer support the old syntax.
 872         self.assert_round_trip('\n', '')
 873         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
 874         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
 875         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 876         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 877         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 878         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')