Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 import unittest
  31
  32 from webkitpy.common.host_mock import MockHost
  33
  34 from webkitpy.layout_tests.models.test_configuration import *
  35 from webkitpy.layout_tests.models.test_expectations import *
  36 from webkitpy.layout_tests.models.test_configuration import *
  37
  38
  39 class MockBugManager(object):
  40     def close_bug(self, bug_id, reference_bug_id=None):
  41         pass
  42
  43     def create_bug(self):
  44         return "BUG_NEWLY_CREATED"
  45
  46
  47 class FunctionsTest(unittest.TestCase):
  48     def test_result_was_expected(self):
  49         # test basics
  50         self.assertEquals(result_was_expected(PASS, set([PASS]),
  51                                               False, False), True)
  52         self.assertEquals(result_was_expected(TEXT, set([PASS]),
  53                                               False, False), False)
  54
  55         # test handling of FAIL expectations
  56         self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
  57                                               False, False), True)
  58         self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
  59                                               False, False), True)
  60         self.assertEquals(result_was_expected(TEXT, set([FAIL]),
  61                                               False, False), True)
  62         self.assertEquals(result_was_expected(CRASH, set([FAIL]),
  63                                               False, False), False)
  64
  65         # test handling of SKIPped tests and results
  66         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
  67                                               False, True), True)
  68         self.assertEquals(result_was_expected(SKIP, set([CRASH]),
  69                                               False, False), False)
  70
  71         # test handling of MISSING results and the REBASELINE modifier
  72         self.assertEquals(result_was_expected(MISSING, set([PASS]),
  73                                               True, False), True)
  74         self.assertEquals(result_was_expected(MISSING, set([PASS]),
  75                                               False, False), False)
  76
  77     def test_remove_pixel_failures(self):
  78         self.assertEquals(remove_pixel_failures(set([TEXT])),
  79                           set([TEXT]))
  80         self.assertEquals(remove_pixel_failures(set([PASS])),
  81                           set([PASS]))
  82         self.assertEquals(remove_pixel_failures(set([IMAGE])),
  83                           set([PASS]))
  84         self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
  85                           set([TEXT]))
  86         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
  87                           set([PASS, CRASH]))
  88
  89
  90 class Base(unittest.TestCase):
  91     # Note that all of these tests are written assuming the configuration
  92     # being tested is Windows XP, Release build.
  93
  94     def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
  95         host = MockHost()
  96         self._port = host.port_factory.get('test-win-xp', None)
  97         self._fs = self._port._filesystem
  98         self._exp = None
  99         unittest.TestCase.__init__(self, testFunc)
 100
 101     def get_test(self, test_name):
 102         # FIXME: Remove this routine and just reference test names directly.
 103         return test_name
 104
 105     def get_basic_tests(self):
 106         return [self.get_test('failures/expected/text.html'),
 107                 self.get_test('failures/expected/image_checksum.html'),
 108                 self.get_test('failures/expected/crash.html'),
 109                 self.get_test('failures/expected/missing_text.html'),
 110                 self.get_test('failures/expected/image.html'),
 111                 self.get_test('passes/text.html')]
 112
 113     def get_basic_expectations(self):
 114         return """
 115 BUG_TEST : failures/expected/text.html = TEXT
 116 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
 117 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
 118 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
 119 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
 120 """
 121
 122     def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
 123         test_config = self._port.test_configuration()
 124         self._exp = TestExpectations(self._port,
 125              tests=self.get_basic_tests(),
 126              expectations=expectations,
 127              test_config=test_config,
 128              is_lint_mode=is_lint_mode,
 129              overrides=overrides)
 130
 131     def assert_exp(self, test, result):
 132         self.assertEquals(self._exp.get_expectations(self.get_test(test)),
 133                           set([result]))
 134
 135
 136 class BasicTests(Base):
 137     def test_basic(self):
 138         self.parse_exp(self.get_basic_expectations())
 139         self.assert_exp('failures/expected/text.html', TEXT)
 140         self.assert_exp('failures/expected/image_checksum.html', IMAGE)
 141         self.assert_exp('passes/text.html', PASS)
 142         self.assert_exp('failures/expected/image.html', PASS)
 143
 144
 145 class MiscTests(Base):
 146     def test_multiple_results(self):
 147         self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
 148         self.assertEqual(self._exp.get_expectations(
 149             self.get_test('failures/expected/text.html')),
 150             set([TEXT, CRASH]))
 151
 152     def test_category_expectations(self):
 153         # This test checks unknown tests are not present in the
 154         # expectations and that known test part of a test category is
 155         # present in the expectations.
 156         exp_str = """
 157 BUGX WONTFIX : failures/expected = IMAGE
 158 """
 159         self.parse_exp(exp_str)
 160         test_name = 'failures/expected/unknown-test.html'
 161         unknown_test = self.get_test(test_name)
 162         self.assertRaises(KeyError, self._exp.get_expectations,
 163                           unknown_test)
 164         self.assert_exp('failures/expected/crash.html', IMAGE)
 165
 166     def test_get_modifiers(self):
 167         self.parse_exp(self.get_basic_expectations())
 168         self.assertEqual(self._exp.get_modifiers(
 169                          self.get_test('passes/text.html')), [])
 170
 171     def test_get_expectations_string(self):
 172         self.parse_exp(self.get_basic_expectations())
 173         self.assertEquals(self._exp.get_expectations_string(
 174                           self.get_test('failures/expected/text.html')),
 175                           'TEXT')
 176
 177     def test_expectation_to_string(self):
 178         # Normal cases are handled by other tests.
 179         self.parse_exp(self.get_basic_expectations())
 180         self.assertRaises(ValueError, self._exp.expectation_to_string,
 181                           -1)
 182
 183     def test_get_test_set(self):
 184         # Handle some corner cases for this routine not covered by other tests.
 185         self.parse_exp(self.get_basic_expectations())
 186         s = self._exp.get_test_set(WONTFIX)
 187         self.assertEqual(s,
 188             set([self.get_test('failures/expected/crash.html'),
 189                  self.get_test('failures/expected/image_checksum.html')]))
 190         s = self._exp.get_test_set(WONTFIX, CRASH)
 191         self.assertEqual(s,
 192             set([self.get_test('failures/expected/crash.html')]))
 193         s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
 194         self.assertEqual(s, set([]))
 195
 196     def test_parse_error_fatal(self):
 197         try:
 198             self.parse_exp("""FOO : failures/expected/text.html = TEXT
 199 SKIP : failures/expected/image.html""")
 200             self.assertFalse(True, "ParseError wasn't raised")
 201         except ParseError, e:
 202             self.assertTrue(e.fatal)
 203             exp_errors = [u"FAILURES FOR %s in LayoutTests/platform/test/test_expectations.txt" % self._port.test_configuration(),
 204                           u"Line:1 Unrecognized modifier 'foo' failures/expected/text.html",
 205                           u"Line:2 Missing expectations SKIP : failures/expected/image.html"]
 206             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
 207             self.assertEqual(e.errors, exp_errors)
 208
 209     def test_parse_error_nonfatal(self):
 210         try:
 211             self.parse_exp('SKIP : failures/expected/text.html = TEXT',
 212                            is_lint_mode=True)
 213             self.assertFalse(True, "ParseError wasn't raised")
 214         except ParseError, e:
 215             self.assertFalse(e.fatal)
 216             exp_errors = [u'FAILURES FOR %s in LayoutTests/platform/test/test_expectations.txt' % self._port.test_configuration(),
 217                           u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
 218             self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
 219             self.assertEqual(e.errors, exp_errors)
 220
 221     def test_overrides(self):
 222         self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
 223                        "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
 224         self.assert_exp('failures/expected/text.html', IMAGE)
 225
 226     def test_overrides__duplicate(self):
 227         self.assertRaises(ParseError, self.parse_exp,
 228              "BUG_EXP: failures/expected/text.html = TEXT",
 229              """
 230 BUG_OVERRIDE : failures/expected/text.html = IMAGE
 231 BUG_OVERRIDE : failures/expected/text.html = CRASH
 232 """)
 233
 234     def test_pixel_tests_flag(self):
 235         def match(test, result, pixel_tests_enabled):
 236             return self._exp.matches_an_expected_result(
 237                 self.get_test(test), result, pixel_tests_enabled)
 238
 239         self.parse_exp(self.get_basic_expectations())
 240         self.assertTrue(match('failures/expected/text.html', TEXT, True))
 241         self.assertTrue(match('failures/expected/text.html', TEXT, False))
 242         self.assertFalse(match('failures/expected/text.html', CRASH, True))
 243         self.assertFalse(match('failures/expected/text.html', CRASH, False))
 244         self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
 245                               True))
 246         self.assertTrue(match('failures/expected/image_checksum.html', PASS,
 247                               False))
 248         self.assertTrue(match('failures/expected/crash.html', SKIP, False))
 249         self.assertTrue(match('passes/text.html', PASS, False))
 250
 251     def test_more_specific_override_resets_skip(self):
 252         self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
 253                        "BUGX : failures/expected/text.html = IMAGE\n")
 254         self.assert_exp('failures/expected/text.html', IMAGE)
 255         self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
 256                                                      'failures/expected/text.html') in
 257                          self._exp.get_tests_with_result_type(SKIP))
 258
 259 class ExpectationSyntaxTests(Base):
 260     def test_missing_expectation(self):
 261         # This is missing the expectation.
 262         self.assertRaises(ParseError, self.parse_exp,
 263                           'BUG_TEST: failures/expected/text.html')
 264
 265     def test_missing_colon(self):
 266         # This is missing the modifiers and the ':'
 267         self.assertRaises(ParseError, self.parse_exp,
 268                           'failures/expected/text.html = TEXT')
 269
 270     def disabled_test_too_many_colons(self):
 271         # FIXME: Enable this test and fix the underlying bug.
 272         self.assertRaises(ParseError, self.parse_exp,
 273                           'BUG_TEST: failures/expected/text.html = PASS :')
 274
 275     def test_too_many_equals_signs(self):
 276         self.assertRaises(ParseError, self.parse_exp,
 277                           'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
 278
 279     def test_unrecognized_expectation(self):
 280         self.assertRaises(ParseError, self.parse_exp,
 281                           'BUG_TEST: failures/expected/text.html = UNKNOWN')
 282
 283     def test_macro(self):
 284         exp_str = """
 285 BUG_TEST WIN : failures/expected/text.html = TEXT
 286 """
 287         self.parse_exp(exp_str)
 288         self.assert_exp('failures/expected/text.html', TEXT)
 289
 290
 291 class SemanticTests(Base):
 292     def test_bug_format(self):
 293         self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
 294
 295     def test_missing_bugid(self):
 296         # This should log a non-fatal error.
 297         self.parse_exp('SLOW : failures/expected/text.html = TEXT')
 298         self.assertTrue(self._exp.has_warnings())
 299
 300     def test_slow_and_timeout(self):
 301         # A test cannot be SLOW and expected to TIMEOUT.
 302         self.assertRaises(ParseError, self.parse_exp,
 303             'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
 304
 305     def test_rebaseline(self):
 306         # Can't lint a file w/ 'REBASELINE' in it.
 307         self.assertRaises(ParseError, self.parse_exp,
 308             'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
 309             is_lint_mode=True)
 310
 311     def test_duplicates(self):
 312         self.assertRaises(ParseError, self.parse_exp, """
 313 BUG_EXP : failures/expected/text.html = TEXT
 314 BUG_EXP : failures/expected/text.html = IMAGE""")
 315
 316         self.assertRaises(ParseError, self.parse_exp,
 317             self.get_basic_expectations(), overrides="""
 318 BUG_OVERRIDE : failures/expected/text.html = TEXT
 319 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
 320
 321     def test_missing_file(self):
 322         # This should log a non-fatal error.
 323         self.parse_exp('BUG_TEST : missing_file.html = TEXT')
 324         self.assertTrue(self._exp.has_warnings(), 1)
 325
 326
 327 class PrecedenceTests(Base):
 328     def test_file_over_directory(self):
 329         # This tests handling precedence of specific lines over directories
 330         # and tests expectations covering entire directories.
 331         exp_str = """
 332 BUGX : failures/expected/text.html = TEXT
 333 BUGX WONTFIX : failures/expected = IMAGE
 334 """
 335         self.parse_exp(exp_str)
 336         self.assert_exp('failures/expected/text.html', TEXT)
 337         self.assert_exp('failures/expected/crash.html', IMAGE)
 338
 339         exp_str = """
 340 BUGX WONTFIX : failures/expected = IMAGE
 341 BUGX : failures/expected/text.html = TEXT
 342 """
 343         self.parse_exp(exp_str)
 344         self.assert_exp('failures/expected/text.html', TEXT)
 345         self.assert_exp('failures/expected/crash.html', IMAGE)
 346
 347     def test_ambiguous(self):
 348         self.assertRaises(ParseError, self.parse_exp, """
 349 BUG_TEST RELEASE : passes/text.html = PASS
 350 BUG_TEST WIN : passes/text.html = FAIL
 351 """)
 352
 353     def test_more_modifiers(self):
 354         exp_str = """
 355 BUG_TEST RELEASE : passes/text.html = PASS
 356 BUG_TEST WIN RELEASE : passes/text.html = TEXT
 357 """
 358         self.assertRaises(ParseError, self.parse_exp, exp_str)
 359
 360     def test_order_in_file(self):
 361         exp_str = """
 362 BUG_TEST WIN RELEASE : passes/text.html = TEXT
 363 BUG_TEST RELEASE : passes/text.html = PASS
 364 """
 365         self.assertRaises(ParseError, self.parse_exp, exp_str)
 366
 367     def test_macro_overrides(self):
 368         exp_str = """
 369 BUG_TEST WIN : passes/text.html = PASS
 370 BUG_TEST XP : passes/text.html = TEXT
 371 """
 372         self.assertRaises(ParseError, self.parse_exp, exp_str)
 373
 374
 375 class RebaseliningTest(Base):
 376     """Test rebaselining-specific functionality."""
 377     def assertRemove(self, input_expectations, tests, expected_expectations):
 378         self.parse_exp(input_expectations)
 379         actual_expectations = self._exp.remove_rebaselined_tests(tests)
 380         self.assertEqual(expected_expectations, actual_expectations)
 381
 382     def test_remove(self):
 383         self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
 384                           'BUGY : failures/expected/image.html = IMAGE\n'
 385                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
 386                           ['failures/expected/text.html'],
 387                           'BUGY : failures/expected/image.html = IMAGE\n'
 388                           'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
 389
 390     def test_no_get_rebaselining_failures(self):
 391         self.parse_exp(self.get_basic_expectations())
 392         self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
 393
 394
 395 class TestExpectationParserTests(unittest.TestCase):
 396     def test_tokenize_blank(self):
 397         expectation = TestExpectationParser.tokenize('')
 398         self.assertEqual(expectation.is_malformed(), False)
 399         self.assertEqual(expectation.comment, None)
 400         self.assertEqual(len(expectation.errors), 0)
 401
 402     def test_tokenize_missing_colon(self):
 403         expectation = TestExpectationParser.tokenize('Qux.')
 404         self.assertEqual(expectation.is_malformed(), True)
 405         self.assertEqual(str(expectation.errors), '["Missing a \':\'"]')
 406
 407     def test_tokenize_extra_colon(self):
 408         expectation = TestExpectationParser.tokenize('FOO : : bar')
 409         self.assertEqual(expectation.is_malformed(), True)
 410         self.assertEqual(str(expectation.errors), '["Extraneous \':\'"]')
 411
 412     def test_tokenize_empty_comment(self):
 413         expectation = TestExpectationParser.tokenize('//')
 414         self.assertEqual(expectation.is_malformed(), False)
 415         self.assertEqual(expectation.comment, '')
 416         self.assertEqual(len(expectation.errors), 0)
 417
 418     def test_tokenize_comment(self):
 419         expectation = TestExpectationParser.tokenize('//Qux.')
 420         self.assertEqual(expectation.is_malformed(), False)
 421         self.assertEqual(expectation.comment, 'Qux.')
 422         self.assertEqual(len(expectation.errors), 0)
 423
 424     def test_tokenize_missing_equal(self):
 425         expectation = TestExpectationParser.tokenize('FOO : bar')
 426         self.assertEqual(expectation.is_malformed(), True)
 427         self.assertEqual(str(expectation.errors), "['Missing expectations\']")
 428
 429     def test_tokenize_extra_equal(self):
 430         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ = Qux.')
 431         self.assertEqual(expectation.is_malformed(), True)
 432         self.assertEqual(str(expectation.errors), '["Extraneous \'=\'"]')
 433
 434     def test_tokenize_valid(self):
 435         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ')
 436         self.assertEqual(expectation.is_malformed(), False)
 437         self.assertEqual(expectation.comment, None)
 438         self.assertEqual(len(expectation.errors), 0)
 439
 440     def test_tokenize_valid_with_comment(self):
 441         expectation = TestExpectationParser.tokenize('FOO : bar = BAZ //Qux.')
 442         self.assertEqual(expectation.is_malformed(), False)
 443         self.assertEqual(expectation.comment, 'Qux.')
 444         self.assertEqual(str(expectation.modifiers), '[\'foo\']')
 445         self.assertEqual(str(expectation.expectations), '[\'baz\']')
 446         self.assertEqual(len(expectation.errors), 0)
 447
 448     def test_tokenize_valid_with_multiple_modifiers(self):
 449         expectation = TestExpectationParser.tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
 450         self.assertEqual(expectation.is_malformed(), False)
 451         self.assertEqual(expectation.comment, 'Qux.')
 452         self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
 453         self.assertEqual(str(expectation.expectations), '[\'baz\']')
 454         self.assertEqual(len(expectation.errors), 0)
 455
 456     def test_parse_empty_string(self):
 457         host = MockHost()
 458         test_port = host.port_factory.get('test-win-xp', None)
 459         test_port.test_exists = lambda test: True
 460         test_config = test_port.test_configuration()
 461         full_test_list = []
 462         expectation_line = TestExpectationParser.tokenize('')
 463         parser = TestExpectationParser(test_port, full_test_list, allow_rebaseline_modifier=False)
 464         parser.parse(expectation_line)
 465         self.assertFalse(expectation_line.is_invalid())
 466
 467
 468 class TestExpectationSerializerTests(unittest.TestCase):
 469     def __init__(self, testFunc):
 470         host = MockHost()
 471         test_port = host.port_factory.get('test-win-xp', None)
 472         self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros())
 473         self._serializer = TestExpectationSerializer(self._converter)
 474         unittest.TestCase.__init__(self, testFunc)
 475
 476     def assert_round_trip(self, in_string, expected_string=None):
 477         expectation = TestExpectationParser.tokenize(in_string)
 478         if expected_string is None:
 479             expected_string = in_string
 480         self.assertEqual(expected_string, self._serializer.to_string(expectation))
 481
 482     def assert_list_round_trip(self, in_string, expected_string=None):
 483         expectations = TestExpectationParser.tokenize_list(in_string)
 484         if expected_string is None:
 485             expected_string = in_string
 486         self.assertEqual(expected_string, TestExpectationSerializer.list_to_string(expectations, self._converter))
 487
 488     def test_unparsed_to_string(self):
 489         expectation = TestExpectationLine()
 490         serializer = TestExpectationSerializer()
 491
 492         self.assertEqual(serializer.to_string(expectation), '')
 493         expectation.comment = 'Qux.'
 494         self.assertEqual(serializer.to_string(expectation), '//Qux.')
 495         expectation.name = 'bar'
 496         self.assertEqual(serializer.to_string(expectation), ' : bar =  //Qux.')
 497         expectation.modifiers = ['foo']
 498         self.assertEqual(serializer.to_string(expectation), 'FOO : bar =  //Qux.')
 499         expectation.expectations = ['bAz']
 500         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ //Qux.')
 501         expectation.expectations = ['bAz1', 'baZ2']
 502         self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
 503         expectation.modifiers = ['foo1', 'foO2']
 504         self.assertEqual(serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
 505         expectation.errors.append('Oh the horror.')
 506         self.assertEqual(serializer.to_string(expectation), '')
 507         expectation.original_string = 'Yes it is!'
 508         self.assertEqual(serializer.to_string(expectation), 'Yes it is!')
 509
 510     def test_unparsed_list_to_string(self):
 511         expectation = TestExpectationLine()
 512         expectation.comment = 'Qux.'
 513         expectation.name = 'bar'
 514         expectation.modifiers = ['foo']
 515         expectation.expectations = ['bAz1', 'baZ2']
 516         self.assertEqual(TestExpectationSerializer.list_to_string([expectation]), 'FOO : bar = BAZ1 BAZ2 //Qux.')
 517
 518     def test_parsed_to_string(self):
 519         expectation_line = TestExpectationLine()
 520         expectation_line.parsed_bug_modifiers = ['BUGX']
 521         expectation_line.name = 'test/name/for/realz.html'
 522         expectation_line.parsed_expectations = set([IMAGE])
 523         self.assertEqual(self._serializer.to_string(expectation_line), None)
 524         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release', 'cpu')])
 525         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE')
 526         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release', 'cpu'), TestConfiguration('xp', 'x86', 'release', 'gpu')])
 527         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE : test/name/for/realz.html = IMAGE')
 528         expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release', 'cpu'), TestConfiguration('xp', 'x86', 'debug', 'gpu')])
 529         self.assertEqual(self._serializer.to_string(expectation_line), 'BUGX XP RELEASE CPU : test/name/for/realz.html = IMAGE\nBUGX XP DEBUG GPU : test/name/for/realz.html = IMAGE')
 530
 531     def test_parsed_expectations_string(self):
 532         expectation_line = TestExpectationLine()
 533         expectation_line.parsed_expectations = set([])
 534         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), '')
 535         expectation_line.parsed_expectations = set([IMAGE_PLUS_TEXT])
 536         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'image+text')
 537         expectation_line.parsed_expectations = set([PASS, FAIL])
 538         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
 539         expectation_line.parsed_expectations = set([FAIL, PASS])
 540         self.assertEqual(self._serializer._parsed_expectations_string(expectation_line), 'pass fail')
 541
 542     def test_parsed_modifier_string(self):
 543         expectation_line = TestExpectationLine()
 544         expectation_line.parsed_bug_modifiers = ['garden-o-matic']
 545         expectation_line.parsed_modifiers = ['for', 'the']
 546         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), 'garden-o-matic for the')
 547         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic for the win')
 548         expectation_line.parsed_bug_modifiers = []
 549         expectation_line.parsed_modifiers = []
 550         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, []), '')
 551         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'win')
 552         expectation_line.parsed_bug_modifiers = ['garden-o-matic', 'total', 'is']
 553         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
 554         expectation_line.parsed_bug_modifiers = []
 555         expectation_line.parsed_modifiers = ['garden-o-matic', 'total', 'is']
 556         self.assertEqual(self._serializer._parsed_modifier_string(expectation_line, ['win']), 'garden-o-matic is total win')
 557
 558     def test_format_result(self):
 559         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', 'comment'), 'MODIFIERS : name = EXPECTATIONS //comment')
 560         self.assertEqual(TestExpectationSerializer._format_result('modifiers', 'name', 'expectations', None), 'MODIFIERS : name = EXPECTATIONS')
 561
 562     def test_string_roundtrip(self):
 563         self.assert_round_trip('')
 564         self.assert_round_trip('FOO')
 565         self.assert_round_trip(':')
 566         self.assert_round_trip('FOO :')
 567         self.assert_round_trip('FOO : bar')
 568         self.assert_round_trip('  FOO :')
 569         self.assert_round_trip('    FOO : bar')
 570         self.assert_round_trip('FOO : bar = BAZ')
 571         self.assert_round_trip('FOO : bar = BAZ //Qux.')
 572         self.assert_round_trip('FOO : bar = BAZ // Qux.')
 573         self.assert_round_trip('FOO : bar = BAZ // Qux.     ')
 574         self.assert_round_trip('FOO : bar = BAZ //        Qux.     ')
 575         self.assert_round_trip('FOO : : bar = BAZ')
 576         self.assert_round_trip('FOO : : bar = BAZ')
 577         self.assert_round_trip('FOO : : bar ==== BAZ')
 578         self.assert_round_trip('=')
 579         self.assert_round_trip('//')
 580         self.assert_round_trip('// ')
 581         self.assert_round_trip('// Foo')
 582         self.assert_round_trip('// Foo')
 583         self.assert_round_trip('// Foo :')
 584         self.assert_round_trip('// Foo : =')
 585
 586     def test_list_roundtrip(self):
 587         self.assert_list_round_trip('')
 588         self.assert_list_round_trip('\n')
 589         self.assert_list_round_trip('\n\n')
 590         self.assert_list_round_trip('bar')
 591         self.assert_list_round_trip('bar\n//Qux.')
 592         self.assert_list_round_trip('bar\n//Qux.\n')
 593
 594     def test_reconstitute_only_these(self):
 595         lines = []
 596         reconstitute_only_these = []
 597
 598         def add_line(matching_configurations, reconstitute):
 599             expectation_line = TestExpectationLine()
 600             expectation_line.original_string = "Nay"
 601             expectation_line.parsed_bug_modifiers = ['BUGX']
 602             expectation_line.name = 'Yay'
 603             expectation_line.parsed_expectations = set([IMAGE])
 604             expectation_line.matching_configurations = matching_configurations
 605             lines.append(expectation_line)
 606             if reconstitute:
 607                 reconstitute_only_these.append(expectation_line)
 608
 609         add_line(set([TestConfiguration('xp', 'x86', 'release', 'cpu')]), False)
 610         add_line(set([TestConfiguration('xp', 'x86', 'release', 'cpu'), TestConfiguration('xp', 'x86', 'release', 'gpu')]), True)
 611         add_line(set([TestConfiguration('xp', 'x86', 'release', 'cpu'), TestConfiguration('xp', 'x86', 'debug', 'gpu')]), False)
 612         serialized = TestExpectationSerializer.list_to_string(lines, self._converter)
 613         self.assertEquals(serialized, "BUGX XP RELEASE CPU : Yay = IMAGE\nBUGX XP RELEASE : Yay = IMAGE\nBUGX XP RELEASE CPU : Yay = IMAGE\nBUGX XP DEBUG GPU : Yay = IMAGE")
 614         serialized = TestExpectationSerializer.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these)
 615         self.assertEquals(serialized, "Nay\nBUGX XP RELEASE : Yay = IMAGE\nNay")
 616
 617     def test_string_whitespace_stripping(self):
 618         self.assert_round_trip('\n', '')
 619         self.assert_round_trip('  FOO : bar = BAZ', 'FOO : bar = BAZ')
 620         self.assert_round_trip('FOO    : bar = BAZ', 'FOO : bar = BAZ')
 621         self.assert_round_trip('FOO : bar = BAZ       // Qux.', 'FOO : bar = BAZ // Qux.')
 622         self.assert_round_trip('FOO : bar =        BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
 623         self.assert_round_trip('FOO :       bar =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
 624         self.assert_round_trip('FOO :       bar     =    BAZ // Qux.', 'FOO : bar = BAZ // Qux.')
 625
 626
 627 if __name__ == '__main__':
 628     unittest.main()