src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import unittest
  30
  31 from webkitpy.common.host_mock import MockHost
  32 from webkitpy.common.system.outputcapture import OutputCapture
  33
  34 from webkitpy.layout_tests.models.test_configuration import *
  35 from webkitpy.layout_tests.models.test_expectations import *
  36
  37 try:
  38     from collections import OrderedDict
  39 except ImportError:
  40     # Needed for Python < 2.7
  41     from webkitpy.thirdparty.ordered_dict import OrderedDict
  42
  43
  44 class Base(unittest.TestCase):
  45     # Note that all of these tests are written assuming the configuration
  46     # being tested is Windows XP, Release build.
  47
  48     def __init__(self, testFunc):
  49         host = MockHost()
  50         self._port = host.port_factory.get('test-win-xp', None)
  51         self._exp = None
  52         unittest.TestCase.__init__(self, testFunc)
  53
  54     def get_basic_tests(self):
  55         return ['failures/expected/text.html',
  56                 'failures/expected/image_checksum.html',
  57                 'failures/expected/crash.html',
  58                 'failures/expected/needsrebaseline.html',
  59                 'failures/expected/needsmanualrebaseline.html',
  60                 'failures/expected/missing_text.html',
  61                 'failures/expected/image.html',
  62                 'failures/expected/timeout.html',
  63                 'passes/text.html']
  64
  65
  66     def get_basic_expectations(self):
  67         return """
  68 Bug(test) failures/expected/text.html [ Failure ]
  69 Bug(test) failures/expected/crash.html [ WontFix ]
  70 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ]
  71 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
  72 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ]
  73 Bug(test) failures/expected/image_checksum.html [ WontFix ]
  74 Bug(test) failures/expected/image.html [ WontFix Mac ]
  75 """
  76
  77     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
  78         expectations_dict = OrderedDict()
  79         expectations_dict['expectations'] = expectations
  80         if overrides:
  81             expectations_dict['overrides'] = overrides
  82         self._port.expectations_dict = lambda: expectations_dict
  83         expectations_to_lint = expectations_dict if is_lint_mode else None
  84         self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_dict=expectations_to_lint, is_lint_mode=is_lint_mode)
  85
  86     def assert_exp_list(self, test, results):
  87         self.assertEqual(self._exp.get_expectations(test), set(results))
  88
  89     def assert_exp(self, test, result):
  90         self.assert_exp_list(test, [result])
  91
  92     def assert_bad_expectations(self, expectations, overrides=None):
  93         self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
  94
  95
  96 class BasicTests(Base):
  97     def test_basic(self):
  98         self.parse_exp(self.get_basic_expectations())
  99         self.assert_exp('failures/expected/text.html', FAIL)
 100         self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX, SKIP])
 101         self.assert_exp('passes/text.html', PASS)
 102         self.assert_exp('failures/expected/image.html', PASS)
 103
 104
 105 class MiscTests(Base):
 106     def test_multiple_results(self):
 107         self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]')
 108         self.assertEqual(self._exp.get_expectations('failures/expected/text.html'), set([FAIL, CRASH]))
 109
 110     def test_result_was_expected(self):
 111         # test basics
 112         self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False), True)
 113         self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False), False)
 114
 115         # test handling of SKIPped tests and results
 116         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False), True)
 117         self.assertEqual(TestExpectations.result_was_expected(SKIP, set([LEAK]), test_needs_rebaselining=False), True)
 118
 119         # test handling of MISSING results and the REBASELINE specifier
 120         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True), True)
 121         self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False), False)
 122
 123         self.assertTrue(TestExpectations.result_was_expected(PASS, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 124         self.assertTrue(TestExpectations.result_was_expected(MISSING, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 125         self.assertTrue(TestExpectations.result_was_expected(TEXT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 126         self.assertTrue(TestExpectations.result_was_expected(IMAGE, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 127         self.assertTrue(TestExpectations.result_was_expected(IMAGE_PLUS_TEXT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 128         self.assertTrue(TestExpectations.result_was_expected(AUDIO, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 129         self.assertFalse(TestExpectations.result_was_expected(TIMEOUT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 130         self.assertFalse(TestExpectations.result_was_expected(CRASH, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 131         self.assertFalse(TestExpectations.result_was_expected(LEAK, set([NEEDS_REBASELINE]), test_needs_rebaselining=False))
 132
 133     def test_remove_pixel_failures(self):
 134         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
 135         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS]))
 136         self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS]))
 137         self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL]))
 138         self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH]))
 139
 140     def test_suffixes_for_expectations(self):
 141         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav']))
 142         self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png']))
 143         self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav']))
 144         self.assertEqual(TestExpectations.suffixes_for_expectations(set()), set())
 145
 146     def test_category_expectations(self):
 147         # This test checks unknown tests are not present in the
 148         # expectations and that known test part of a test category is
 149         # present in the expectations.
 150         exp_str = 'Bug(x) failures/expected [ WontFix ]'
 151         self.parse_exp(exp_str)
 152         test_name = 'failures/expected/unknown-test.html'
 153         unknown_test = test_name
 154         self.assertRaises(KeyError, self._exp.get_expectations,
 155                           unknown_test)
 156         self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
 157
 158     def test_get_expectations_string(self):
 159         self.parse_exp(self.get_basic_expectations())
 160         self.assertEqual(self._exp.get_expectations_string('failures/expected/text.html'), 'FAIL')
 161
 162     def test_expectation_to_string(self):
 163         # Normal cases are handled by other tests.
 164         self.parse_exp(self.get_basic_expectations())
 165         self.assertRaises(ValueError, self._exp.expectation_to_string,
 166                           -1)
 167
 168     def test_get_test_set(self):
 169         # Handle some corner cases for this routine not covered by other tests.
 170         self.parse_exp(self.get_basic_expectations())
 171         s = self._exp.get_test_set(WONTFIX)
 172         self.assertEqual(s, set(['failures/expected/crash.html', 'failures/expected/image_checksum.html']))
 173
 174     def test_needs_rebaseline_reftest(self):
 175         try:
 176             filesystem = self._port.host.filesystem
 177             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline.html'), 'content')
 178             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsrebaseline-expected.html'), 'content')
 179             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline.html'), 'content')
 180             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'failures/expected/needsmanualrebaseline-expected.html'), 'content')
 181             self.parse_exp("""Bug(user) failures/expected/needsrebaseline.html [ NeedsRebaseline ]
 182 Bug(user) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]""", is_lint_mode=True)
 183             self.assertFalse(True, "ParseError wasn't raised")
 184         except ParseError, e:
 185             warnings = """expectations:1 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline failures/expected/needsrebaseline.html
 186 expectations:2 A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline failures/expected/needsmanualrebaseline.html"""
 187             self.assertEqual(str(e), warnings)
 188
 189     def test_parse_warning(self):
 190         try:
 191             filesystem = self._port.host.filesystem
 192             filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content')
 193             'disabled-test.html-disabled',
 194             self.parse_exp("Bug(user) [ FOO ] failures/expected/text.html [ Failure ]\n"
 195                 "Bug(user) non-existent-test.html [ Failure ]\n"
 196                 "Bug(user) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True)
 197             self.assertFalse(True, "ParseError wasn't raised")
 198         except ParseError, e:
 199             warnings = ("expectations:1 Unrecognized specifier 'foo' failures/expected/text.html\n"
 200                         "expectations:2 Path does not exist. non-existent-test.html")
 201             self.assertEqual(str(e), warnings)
 202
 203     def test_parse_warnings_are_logged_if_not_in_lint_mode(self):
 204         oc = OutputCapture()
 205         try:
 206             oc.capture_output()
 207             self.parse_exp('-- this should be a syntax error', is_lint_mode=False)
 208         finally:
 209             _, _, logs = oc.restore_output()
 210             self.assertNotEquals(logs, '')
 211
 212     def test_error_on_different_platform(self):
 213         # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
 214         self.assertRaises(ParseError, self.parse_exp,
 215             'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]',
 216             is_lint_mode=True)
 217
 218     def test_error_on_different_build_type(self):
 219         # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode.
 220         self.assertRaises(ParseError, self.parse_exp,
 221             'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]',
 222             is_lint_mode=True)
 223
 224     def test_overrides(self):
 225         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
 226                        "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]")
 227         self.assert_exp_list('failures/expected/text.html', [FAIL, IMAGE])
 228
 229     def test_overrides__directory(self):
 230         self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]",
 231                        "Bug(override) failures/expected [ Crash ]")
 232         self.assert_exp_list('failures/expected/text.html', [FAIL, CRASH])
 233         self.assert_exp_list('failures/expected/image.html', [CRASH])
 234
 235     def test_overrides__duplicate(self):
 236         self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]",
 237                                      "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n"
 238                                      "Bug(override) failures/expected/text.html [ Crash ]\n")
 239
 240     def test_pixel_tests_flag(self):
 241         def match(test, result, pixel_tests_enabled):
 242             return self._exp.matches_an_expected_result(
 243                 test, result, pixel_tests_enabled, sanitizer_is_enabled=False)
 244
 245         self.parse_exp(self.get_basic_expectations())
 246         self.assertTrue(match('failures/expected/text.html', FAIL, True))
 247         self.assertTrue(match('failures/expected/text.html', FAIL, False))
 248         self.assertFalse(match('failures/expected/text.html', CRASH, True))
 249         self.assertFalse(match('failures/expected/text.html', CRASH, False))
 250         self.assertTrue(match('failures/expected/image_checksum.html', PASS, True))
 251         self.assertTrue(match('failures/expected/image_checksum.html', PASS, False))
 252         self.assertTrue(match('failures/expected/crash.html', PASS, False))
 253         self.assertTrue(match('failures/expected/needsrebaseline.html', TEXT, True))
 254         self.assertFalse(match('failures/expected/needsrebaseline.html', CRASH, True))
 255         self.assertTrue(match('failures/expected/needsmanualrebaseline.html', TEXT, True))
 256         self.assertFalse(match('failures/expected/needsmanualrebaseline.html', CRASH, True))
 257         self.assertTrue(match('passes/text.html', PASS, False))
 258
 259     def test_sanitizer_flag(self):
 260         def match(test, result):
 261             return self._exp.matches_an_expected_result(
 262                 test, result, pixel_tests_are_enabled=False, sanitizer_is_enabled=True)
 263
 264         self.parse_exp("""
 265 Bug(test) failures/expected/crash.html [ Crash ]
 266 Bug(test) failures/expected/image.html [ ImageOnlyFailure ]
 267 Bug(test) failures/expected/text.html [ Failure ]
 268 Bug(test) failures/expected/timeout.html [ Timeout ]
 269 """)
 270         self.assertTrue(match('failures/expected/crash.html', CRASH))
 271         self.assertTrue(match('failures/expected/image.html', PASS))
 272         self.assertTrue(match('failures/expected/text.html', PASS))
 273         self.assertTrue(match('failures/expected/timeout.html', TIMEOUT))
 274
 275     def test_more_specific_override_resets_skip(self):
 276         self.parse_exp("Bug(x) failures/expected [ Skip ]\n"
 277                        "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n")
 278         self.assert_exp('failures/expected/text.html', IMAGE)
 279         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
 280                                                      'failures/expected/text.html') in
 281                          self._exp.get_tests_with_result_type(SKIP))
 282
 283     def test_bot_test_expectations(self):
 284         """Test that expectations are merged rather than overridden when using flaky option 'unexpected'."""
 285         test_name1 = 'failures/expected/text.html'
 286         test_name2 = 'passes/text.html'
 287
 288         expectations_dict = OrderedDict()
 289         expectations_dict['expectations'] = "Bug(x) %s [ ImageOnlyFailure ]\nBug(x) %s [ Slow ]\n" % (test_name1, test_name2)
 290         self._port.expectations_dict = lambda: expectations_dict
 291
 292         expectations = TestExpectations(self._port, self.get_basic_tests())
 293         self.assertEqual(expectations.get_expectations(test_name1), set([IMAGE]))
 294         self.assertEqual(expectations.get_expectations(test_name2), set([SLOW]))
 295
 296         def bot_expectations():
 297             return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']}
 298         self._port.bot_expectations = bot_expectations
 299         self._port._options.ignore_flaky_tests = 'unexpected'
 300
 301         expectations = TestExpectations(self._port, self.get_basic_tests())
 302         self.assertEqual(expectations.get_expectations(test_name1), set([PASS, IMAGE, TIMEOUT]))
 303         self.assertEqual(expectations.get_expectations(test_name2), set([CRASH, SLOW]))
 304
 305 class SkippedTests(Base):
 306     def check(self, expectations, overrides, skips, lint=False, expected_results=[WONTFIX, SKIP, FAIL]):
 307         port = MockHost().port_factory.get('test-win-xp')
 308         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo')
 309         expectations_dict = OrderedDict()
 310         expectations_dict['expectations'] = expectations
 311         if overrides:
 312             expectations_dict['overrides'] = overrides
 313         port.expectations_dict = lambda: expectations_dict
 314         port.skipped_layout_tests = lambda tests: set(skips)
 315         expectations_to_lint = expectations_dict if lint else None
 316         exp = TestExpectations(port, ['failures/expected/text.html'], expectations_dict=expectations_to_lint, is_lint_mode=lint)
 317         self.assertEqual(exp.get_expectations('failures/expected/text.html'), set(expected_results))
 318
 319     def test_skipped_tests_work(self):
 320         self.check(expectations='', overrides=None, skips=['failures/expected/text.html'], expected_results=[WONTFIX, SKIP])
 321
 322     def test_duplicate_skipped_test_fails_lint(self):
 323         self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n',
 324             overrides=None, skips=['failures/expected/text.html'], lint=True)
 325
 326     def test_skipped_file_overrides_expectations(self):
 327         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
 328                    skips=['failures/expected/text.html'])
 329
 330     def test_skipped_dir_overrides_expectations(self):
 331         self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None,
 332                    skips=['failures/expected'])
 333
 334     def test_skipped_file_overrides_overrides(self):
 335         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
 336                    skips=['failures/expected/text.html'])
 337
 338     def test_skipped_dir_overrides_overrides(self):
 339         self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n',
 340                    skips=['failures/expected'])
 341
 342     def test_skipped_entry_dont_exist(self):
 343         port = MockHost().port_factory.get('test-win-xp')
 344         expectations_dict = OrderedDict()
 345         expectations_dict['expectations'] = ''
 346         port.expectations_dict = lambda: expectations_dict
 347         port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html'])
 348         capture = OutputCapture()
 349         capture.capture_output()
 350         exp = TestExpectations(port)
 351         _, _, logs = capture.restore_output()
 352         self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs)
 353
 354     def test_expectations_string(self):
 355         self.parse_exp(self.get_basic_expectations())
 356         notrun = 'failures/expected/text.html'
 357         self._exp.add_extra_skipped_tests([notrun])
 358         self.assertEqual('NOTRUN', self._exp.get_expectations_string(notrun))
 359
 360
 361 class ExpectationSyntaxTests(Base):
 362     def test_unrecognized_expectation(self):
 363         self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]')
 364
 365     def test_macro(self):
 366         exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]'
 367         self.parse_exp(exp_str)
 368         self.assert_exp('failures/expected/text.html', FAIL)
 369
 370     def assert_tokenize_exp(self, line, bugs=None, specifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
 371         bugs = bugs or []
 372         specifiers = specifiers or []
 373         expectations = expectations or []
 374         warnings = warnings or []
 375         filename = 'TestExpectations'
 376         line_number = '1'
 377         expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number)
 378         self.assertEqual(expectation_line.warnings, warnings)
 379         self.assertEqual(expectation_line.name, name)
 380         self.assertEqual(expectation_line.filename, filename)
 381         self.assertEqual(expectation_line.line_numbers, line_number)
 382         if not warnings:
 383             self.assertEqual(expectation_line.specifiers, specifiers)
 384             self.assertEqual(expectation_line.expectations, expectations)
 385
 386     def test_comments(self):
 387         self.assert_tokenize_exp("# comment", name=None, comment="# comment")
 388         self.assert_tokenize_exp("foo.html [ Pass ] # comment", comment="# comment", expectations=['PASS'], specifiers=[])
 389
 390     def test_config_specifiers(self):
 391         self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', specifiers=['MAC'], expectations=['FAIL'])
 392
 393     def test_unknown_config(self):
 394         self.assert_tokenize_exp('[ Foo ] foo.html [ Pass ]', specifiers=['Foo'], expectations=['PASS'])
 395
 396     def test_unknown_expectation(self):
 397         self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"'])
 398
 399     def test_skip(self):
 400         self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectations=['SKIP'])
 401
 402     def test_slow(self):
 403         self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectations=['SLOW'])
 404
 405     def test_wontfix(self):
 406         self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectations=['WONTFIX', 'SKIP'])
 407         self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', specifiers=[], expectations=['WONTFIX', 'SKIP'],
 408             warnings=['A test marked Skip or WontFix must not have other expectations.'])
 409
 410     def test_blank_line(self):
 411         self.assert_tokenize_exp('', name=None)
 412
 413     def test_warnings(self):
 414         self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.', 'Missing expectations.'], name=None)
 415         self.assert_tokenize_exp('[ [', warnings=['unexpected "["', 'Missing expectations.'], name=None)
 416         self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"', 'Missing expectations.'], name=None)
 417
 418         self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.', 'Missing expectations.'])
 419         self.assert_tokenize_exp('foo.html', warnings=['Missing expectations.'])
 420
 421
 422 class SemanticTests(Base):
 423     def test_bug_format(self):
 424         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
 425
 426     def test_bad_bugid(self):
 427         try:
 428             self.parse_exp('crbug/1234 failures/expected/text.html [ Failure ]', is_lint_mode=True)
 429             self.fail('should have raised an error about a bad bug identifier')
 430         except ParseError, exp:
 431             self.assertEqual(len(exp.warnings), 3)
 432
 433     def test_missing_bugid(self):
 434         self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=False)
 435         self.assertFalse(self._exp.has_warnings())
 436
 437         try:
 438             self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=True)
 439         except ParseError, exp:
 440             self.assertEqual(exp.warnings, ['expectations:1 Test lacks BUG specifier. failures/expected/text.html'])
 441
 442     def test_skip_and_wontfix(self):
 443         # Skip is not allowed to have other expectations as well, because those
 444         # expectations won't be exercised and may become stale .
 445         self.parse_exp('failures/expected/text.html [ Failure Skip ]')
 446         self.assertTrue(self._exp.has_warnings())
 447
 448         self.parse_exp('failures/expected/text.html [ Crash WontFix ]')
 449         self.assertTrue(self._exp.has_warnings())
 450
 451         self.parse_exp('failures/expected/text.html [ Pass WontFix ]')
 452         self.assertTrue(self._exp.has_warnings())
 453
 454     def test_rebaseline(self):
 455         # Can't lint a file w/ 'REBASELINE' in it.
 456         self.assertRaises(ParseError, self.parse_exp,
 457             'Bug(test) failures/expected/text.html [ Failure Rebaseline ]',
 458             is_lint_mode=True)
 459
 460     def test_duplicates(self):
 461         self.assertRaises(ParseError, self.parse_exp, """
 462 Bug(exp) failures/expected/text.html [ Failure ]
 463 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
 464
 465         self.assertRaises(ParseError, self.parse_exp,
 466             self.get_basic_expectations(), overrides="""
 467 Bug(override) failures/expected/text.html [ Failure ]
 468 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True)
 469
 470     def test_duplicate_with_line_before_preceding_line(self):
 471         self.assert_bad_expectations("""Bug(exp) [ Debug ] failures/expected/text.html [ Failure ]
 472 Bug(exp) [ Release ] failures/expected/text.html [ Failure ]
 473 Bug(exp) [ Debug ] failures/expected/text.html [ Failure ]
 474 """)
 475
 476     def test_missing_file(self):
 477         self.parse_exp('Bug(test) missing_file.html [ Failure ]')
 478         self.assertTrue(self._exp.has_warnings(), 1)
 479
 480
 481 class PrecedenceTests(Base):
 482     def test_file_over_directory(self):
 483         # This tests handling precedence of specific lines over directories
 484         # and tests expectations covering entire directories.
 485         exp_str = """
 486 Bug(x) failures/expected/text.html [ Failure ]
 487 Bug(y) failures/expected [ WontFix ]
 488 """
 489         self.parse_exp(exp_str)
 490         self.assert_exp('failures/expected/text.html', FAIL)
 491         self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
 492
 493         exp_str = """
 494 Bug(x) failures/expected [ WontFix ]
 495 Bug(y) failures/expected/text.html [ Failure ]
 496 """
 497         self.parse_exp(exp_str)
 498         self.assert_exp('failures/expected/text.html', FAIL)
 499         self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP])
 500
 501     def test_ambiguous(self):
 502         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
 503                                      "Bug(test) [ Win ] passes/text.html [ Failure ]\n")
 504
 505     def test_more_specifiers(self):
 506         self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n"
 507                                      "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n")
 508
 509     def test_order_in_file(self):
 510         self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n"
 511                                      "Bug(test) [ Release ] : passes/text.html [ Pass ]\n")
 512
 513     def test_macro_overrides(self):
 514         self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n"
 515                                      "Bug(test) [ XP ] passes/text.html [ Failure ]\n")
 516
 517
 518 class RemoveConfigurationsTest(Base):
 519     def test_remove(self):
 520         host = MockHost()
 521         test_port = host.port_factory.get('test-win-xp', None)
 522         test_port.test_exists = lambda test: True
 523         test_port.test_isfile = lambda test: True
 524
 525         test_config = test_port.test_configuration()
 526         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ]
 527 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 528 """}
 529         expectations = TestExpectations(test_port, self.get_basic_tests())
 530
 531         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 532
 533         self.assertEqual("""Bug(x) [ Linux Win7 Release ] failures/expected/foo.html [ Failure ]
 534 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 535 """, actual_expectations)
 536
 537     def test_remove_needs_rebaseline(self):
 538         host = MockHost()
 539         test_port = host.port_factory.get('test-win-xp', None)
 540         test_port.test_exists = lambda test: True
 541         test_port.test_isfile = lambda test: True
 542
 543         test_config = test_port.test_configuration()
 544         test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Win ] failures/expected/foo.html [ NeedsRebaseline ]
 545 """}
 546         expectations = TestExpectations(test_port, self.get_basic_tests())
 547
 548         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 549
 550         self.assertEqual("""Bug(x) [ XP Debug ] failures/expected/foo.html [ NeedsRebaseline ]
 551 Bug(x) [ Win7 ] failures/expected/foo.html [ NeedsRebaseline ]
 552 """, actual_expectations)
 553
 554     def test_remove_multiple_configurations(self):
 555         host = MockHost()
 556         test_port = host.port_factory.get('test-win-xp', None)
 557         test_port.test_exists = lambda test: True
 558         test_port.test_isfile = lambda test: True
 559
 560         test_config = test_port.test_configuration()
 561         test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 562 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 563 """}
 564         expectations = TestExpectations(test_port)
 565
 566         actual_expectations = expectations.remove_configurations([
 567             ('failures/expected/foo.html', test_config),
 568             ('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()),
 569         ])
 570
 571         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 572 """, actual_expectations)
 573
 574     def test_remove_line_with_comments(self):
 575         host = MockHost()
 576         test_port = host.port_factory.get('test-win-xp', None)
 577         test_port.test_exists = lambda test: True
 578         test_port.test_isfile = lambda test: True
 579
 580         test_config = test_port.test_configuration()
 581         test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 582
 583  # This comment line should get stripped. As should the preceding line.
 584 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 585 """}
 586         expectations = TestExpectations(test_port)
 587
 588         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 589         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())])
 590
 591         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 592 """, actual_expectations)
 593
 594     def test_remove_line_with_comments_at_start(self):
 595         host = MockHost()
 596         test_port = host.port_factory.get('test-win-xp', None)
 597         test_port.test_exists = lambda test: True
 598         test_port.test_isfile = lambda test: True
 599
 600         test_config = test_port.test_configuration()
 601         test_port.expectations_dict = lambda: {'expectations': """
 602  # This comment line should get stripped. As should the preceding line.
 603 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 604
 605 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 606 """}
 607         expectations = TestExpectations(test_port)
 608
 609         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 610         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())])
 611
 612         self.assertEqual("""
 613 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 614 """, actual_expectations)
 615
 616     def test_remove_line_with_comments_at_end_with_no_trailing_newline(self):
 617         host = MockHost()
 618         test_port = host.port_factory.get('test-win-xp', None)
 619         test_port.test_exists = lambda test: True
 620         test_port.test_isfile = lambda test: True
 621
 622         test_config = test_port.test_configuration()
 623         test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 624
 625  # This comment line should get stripped. As should the preceding line.
 626 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]"""}
 627         expectations = TestExpectations(test_port)
 628
 629         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 630         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())])
 631
 632         self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]""", actual_expectations)
 633
 634     def test_remove_line_leaves_comments_for_next_line(self):
 635         host = MockHost()
 636         test_port = host.port_factory.get('test-win-xp', None)
 637         test_port.test_exists = lambda test: True
 638         test_port.test_isfile = lambda test: True
 639
 640         test_config = test_port.test_configuration()
 641         test_port.expectations_dict = lambda: {'expectations': """
 642  # This comment line should not get stripped.
 643 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 644 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 645 """}
 646         expectations = TestExpectations(test_port)
 647
 648         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 649         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())])
 650
 651         self.assertEqual("""
 652  # This comment line should not get stripped.
 653 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 654 """, actual_expectations)
 655
 656     def test_remove_line_no_whitespace_lines(self):
 657         host = MockHost()
 658         test_port = host.port_factory.get('test-win-xp', None)
 659         test_port.test_exists = lambda test: True
 660         test_port.test_isfile = lambda test: True
 661
 662         test_config = test_port.test_configuration()
 663         test_port.expectations_dict = lambda: {'expectations': """
 664  # This comment line should get stripped.
 665 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 666  # This comment line should not get stripped.
 667 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 668 """}
 669         expectations = TestExpectations(test_port)
 670
 671         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 672         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())])
 673
 674         self.assertEqual(""" # This comment line should not get stripped.
 675 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 676 """, actual_expectations)
 677
 678     def test_remove_first_line(self):
 679         host = MockHost()
 680         test_port = host.port_factory.get('test-win-xp', None)
 681         test_port.test_exists = lambda test: True
 682         test_port.test_isfile = lambda test: True
 683
 684         test_config = test_port.test_configuration()
 685         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]
 686  # This comment line should not get stripped.
 687 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 688 """}
 689         expectations = TestExpectations(test_port)
 690
 691         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 692         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())])
 693
 694         self.assertEqual(""" # This comment line should not get stripped.
 695 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]
 696 """, actual_expectations)
 697
 698     def test_remove_flaky_line(self):
 699         host = MockHost()
 700         test_port = host.port_factory.get('test-win-xp', None)
 701         test_port.test_exists = lambda test: True
 702         test_port.test_isfile = lambda test: True
 703
 704         test_config = test_port.test_configuration()
 705         test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ] failures/expected/foo.html [ Failure Timeout ]
 706 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ]
 707 """}
 708         expectations = TestExpectations(test_port)
 709
 710         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
 711         actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())])
 712
 713         self.assertEqual("""Bug(x) [ Win Debug ] failures/expected/foo.html [ Failure Timeout ]
 714 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ]
 715 """, actual_expectations)
 716
 717
 718 class RebaseliningTest(Base):
 719     def test_get_rebaselining_failures(self):
 720         # Make sure we find a test as needing a rebaseline even if it is not marked as a failure.
 721         self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n')
 722         self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
 723
 724         self.parse_exp(self.get_basic_expectations())
 725         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
 726
 727
 728 class TestExpectationsParserTests(unittest.TestCase):
 729     def __init__(self, testFunc):
 730         host = MockHost()
 731         test_port = host.port_factory.get('test-win-xp', None)
 732         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
 733         unittest.TestCase.__init__(self, testFunc)
 734         self._parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], is_lint_mode=False)
 735
 736     def test_expectation_line_for_test(self):
 737         # This is kind of a silly test, but it at least ensures that we don't throw an error.
 738         test_name = 'foo/test.html'
 739         expectations = set(["PASS", "IMAGE"])
 740
 741         expectation_line = TestExpectationLine()
 742         expectation_line.original_string = test_name
 743         expectation_line.name = test_name
 744         expectation_line.filename = '<Bot TestExpectations>'
 745         expectation_line.line_numbers = '0'
 746         expectation_line.expectations = expectations
 747         self._parser._parse_line(expectation_line)
 748
 749         self.assertEqual(self._parser.expectation_line_for_test(test_name, expectations), expectation_line)
 750
 751
 752 class TestExpectationSerializationTests(unittest.TestCase):
 753     def __init__(self, testFunc):
 754         host = MockHost()
 755         test_port = host.port_factory.get('test-win-xp', None)
 756         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
 757         unittest.TestCase.__init__(self, testFunc)
 758
 759     def _tokenize(self, line):
 760         return TestExpectationParser._tokenize_line('path', line, 0)
 761
 762     def assert_round_trip(self, in_string, expected_string=None):
 763         expectation = self._tokenize(in_string)
 764         if expected_string is None:
 765             expected_string = in_string
 766         self.assertEqual(expected_string, expectation.to_string(self._converter))
 767
 768     def assert_list_round_trip(self, in_string, expected_string=None):
 769         host = MockHost()
 770         parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], is_lint_mode=False)
 771         expectations = parser.parse('path', in_string)
 772         if expected_string is None:
 773             expected_string = in_string
 774         self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter))
 775
 776     def test_unparsed_to_string(self):
 777         expectation = TestExpectationLine()
 778
 779         self.assertEqual(expectation.to_string(self._converter), '')
 780         expectation.comment = ' Qux.'
 781         self.assertEqual(expectation.to_string(self._converter), '# Qux.')
 782         expectation.name = 'bar'
 783         self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.')
 784         expectation.specifiers = ['foo']
 785         # FIXME: case should be preserved here but we can't until we drop the old syntax.
 786         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.')
 787         expectation.expectations = ['bAz']
 788         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.')
 789         expectation.expectations = ['bAz1', 'baZ2']
 790         self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.')
 791         expectation.specifiers = ['foo1', 'foO2']
 792         self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.')
 793         expectation.warnings.append('Oh the horror.')
 794         self.assertEqual(expectation.to_string(self._converter), '')
 795         expectation.original_string = 'Yes it is!'
 796         self.assertEqual(expectation.to_string(self._converter), 'Yes it is!')
 797
 798     def test_unparsed_list_to_string(self):
 799         expectation = TestExpectationLine()
 800         expectation.comment = 'Qux.'
 801         expectation.name = 'bar'
 802         expectation.specifiers = ['foo']
 803         expectation.expectations = ['bAz1', 'baZ2']
 804         # FIXME: case should be preserved here but we can't until we drop the old syntax.
 805         self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.')
 806
 807     def test_parsed_to_string(self):
 808         expectation_line = TestExpectationLine()
 809         expectation_line.bugs = ['Bug(x)']
 810         expectation_line.name = 'test/name/for/realz.html'
 811         expectation_line.parsed_expectations = set([IMAGE])
 812         self.assertEqual(expectation_line.to_string(self._converter), None)
 813         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')])
 814         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]')
 815         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')])
 816         self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]')
 817
 818     def test_serialize_parsed_expectations(self):
 819         expectation_line = TestExpectationLine()
 820         expectation_line.parsed_expectations = set([])
 821         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
 822         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '')
 823         expectation_line.parsed_expectations = set([FAIL])
 824         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail')
 825         expectation_line.parsed_expectations = set([PASS, IMAGE])
 826         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'image pass')
 827         expectation_line.parsed_expectations = set([FAIL, PASS])
 828         self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail')
 829
 830     def test_serialize_parsed_specifier_string(self):
 831         expectation_line = TestExpectationLine()
 832         expectation_line.bugs = ['garden-o-matic']
 833         expectation_line.parsed_specifiers = ['the', 'for']
 834         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, []), 'for the')
 835         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, ['win']), 'for the win')
 836         expectation_line.bugs = []
 837         expectation_line.parsed_specifiers = []
 838         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, []), '')
 839         self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, ['win']), 'win')
 840
 841     def test_format_line(self):
 842         self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment')
 843         self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]')
 844
 845     def test_string_roundtrip(self):
 846         self.assert_round_trip('')
 847         self.assert_round_trip('[')
 848         self.assert_round_trip('FOO [')
 849         self.assert_round_trip('FOO ] bar')
 850         self.assert_round_trip('  FOO [')
 851         self.assert_round_trip('    [ FOO ] ')
 852         self.assert_round_trip('[ FOO ] bar [ BAZ ]')
 853         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
 854         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.')
 855         self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.     ')
 856         self.assert_round_trip('[ FOO ] bar [ BAZ ] #        Qux.     ')
 857         self.assert_round_trip('[ FOO ] ] ] bar BAZ')
 858         self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]')
 859         self.assert_round_trip('FOO ] ] bar ==== BAZ')
 860         self.assert_round_trip('=')
 861         self.assert_round_trip('#')
 862         self.assert_round_trip('# ')
 863         self.assert_round_trip('# Foo')
 864         self.assert_round_trip('# Foo')
 865         self.assert_round_trip('# Foo :')
 866         self.assert_round_trip('# Foo : =')
 867
 868     def test_list_roundtrip(self):
 869         self.assert_list_round_trip('')
 870         self.assert_list_round_trip('\n')
 871         self.assert_list_round_trip('\n\n')
 872         self.assert_list_round_trip('bar')
 873         self.assert_list_round_trip('bar\n# Qux.')
 874         self.assert_list_round_trip('bar\n# Qux.\n')
 875
 876     def test_reconstitute_only_these(self):
 877         lines = []
 878         reconstitute_only_these = []
 879
 880         def add_line(matching_configurations, reconstitute):
 881             expectation_line = TestExpectationLine()
 882             expectation_line.original_string = "Nay"
 883             expectation_line.bugs = ['Bug(x)']
 884             expectation_line.name = 'Yay'
 885             expectation_line.parsed_expectations = set([IMAGE])
 886             expectation_line.matching_configurations = matching_configurations
 887             lines.append(expectation_line)
 888             if reconstitute:
 889                 reconstitute_only_these.append(expectation_line)
 890
 891         add_line(set([TestConfiguration('xp', 'x86', 'release')]), True)
 892         add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False)
 893         serialized = TestExpectations.list_to_string(lines, self._converter)
 894         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]")
 895         serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
 896         self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay")
 897
 898     def disabled_test_string_whitespace_stripping(self):
 899         # FIXME: Re-enable this test once we rework the code to no longer support the old syntax.
 900         self.assert_round_trip('\n', '')
 901         self.assert_round_trip('  [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
 902         self.assert_round_trip('[ FOO ]    bar [ BAZ ]', '[ FOO ] bar [ BAZ ]')
 903         self.assert_round_trip('[ FOO ] bar [ BAZ ]       # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 904         self.assert_round_trip('[ FOO ] bar [        BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 905         self.assert_round_trip('[ FOO ]       bar [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')
 906         self.assert_round_trip('[ FOO ]       bar     [    BAZ ]  # Qux.', '[ FOO ] bar [ BAZ ] # Qux.')