src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 """A helper class for reading in and dealing with tests expectations
  30 for layout tests.
  31 """
  32
  33 import logging
  34 import re
  35
  36 from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
  37
  38 _log = logging.getLogger(__name__)
  39
  40
  41 # Test expectation and specifier constants.
  42 #
  43 # FIXME: range() starts with 0 which makes if expectation checks harder
  44 # as PASS is 0.
  45 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WONTFIX,
  46  SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(19)
  47
  48 # FIXME: Perhas these two routines should be part of the Port instead?
  49 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
  50
  51 WEBKIT_BUG_PREFIX = 'webkit.org/b/'
  52 CHROMIUM_BUG_PREFIX = 'crbug.com/'
  53 V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id='
  54 NAMED_BUG_PREFIX = 'Bug('
  55
  56 MISSING_KEYWORD = 'Missing'
  57 NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline'
  58 NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline'
  59
  60 class ParseError(Exception):
  61     def __init__(self, warnings):
  62         super(ParseError, self).__init__()
  63         self.warnings = warnings
  64
  65     def __str__(self):
  66         return '\n'.join(map(str, self.warnings))
  67
  68     def __repr__(self):
  69         return 'ParseError(warnings=%s)' % self.warnings
  70
  71
  72 class TestExpectationParser(object):
  73     """Provides parsing facilities for lines in the test_expectation.txt file."""
  74
  75     # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the case studdly-caps to match the actual file contents.
  76     REBASELINE_MODIFIER = 'rebaseline'
  77     NEEDS_REBASELINE_MODIFIER = 'needsrebaseline'
  78     NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline'
  79     PASS_EXPECTATION = 'pass'
  80     SKIP_MODIFIER = 'skip'
  81     SLOW_MODIFIER = 'slow'
  82     WONTFIX_MODIFIER = 'wontfix'
  83
  84     TIMEOUT_EXPECTATION = 'timeout'
  85
  86     MISSING_BUG_WARNING = 'Test lacks BUG specifier.'
  87
  88     def __init__(self, port, full_test_list, is_lint_mode):
  89         self._port = port
  90         self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
  91         self._full_test_list = full_test_list
  92         self._is_lint_mode = is_lint_mode
  93
  94     def parse(self, filename, expectations_string):
  95         expectation_lines = []
  96         line_number = 0
  97         for line in expectations_string.split("\n"):
  98             line_number += 1
  99             test_expectation = self._tokenize_line(filename, line, line_number)
 100             self._parse_line(test_expectation)
 101             expectation_lines.append(test_expectation)
 102         return expectation_lines
 103
 104     def _create_expectation_line(self, test_name, expectations, file_name):
 105         expectation_line = TestExpectationLine()
 106         expectation_line.original_string = test_name
 107         expectation_line.name = test_name
 108         expectation_line.filename = file_name
 109         expectation_line.expectations = expectations
 110         return expectation_line
 111
 112     def expectation_line_for_test(self, test_name, expectations):
 113         expectation_line = self._create_expectation_line(test_name, expectations, '<Bot TestExpectations>')
 114         self._parse_line(expectation_line)
 115         return expectation_line
 116
 117
 118     def expectation_for_skipped_test(self, test_name):
 119         if not self._port.test_exists(test_name):
 120             _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
 121         expectation_line = self._create_expectation_line(test_name, [TestExpectationParser.PASS_EXPECTATION], '<Skipped file>')
 122         expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER]
 123         expectation_line.is_skipped_outside_expectations_file = True
 124         self._parse_line(expectation_line)
 125         return expectation_line
 126
 127     def _parse_line(self, expectation_line):
 128         if not expectation_line.name:
 129             return
 130
 131         if not self._check_test_exists(expectation_line):
 132             return
 133
 134         expectation_line.is_file = self._port.test_isfile(expectation_line.name)
 135         if expectation_line.is_file:
 136             expectation_line.path = expectation_line.name
 137         else:
 138             expectation_line.path = self._port.normalize_test_name(expectation_line.name)
 139
 140         self._collect_matching_tests(expectation_line)
 141
 142         self._parse_specifiers(expectation_line)
 143         self._parse_expectations(expectation_line)
 144
 145     def _parse_specifiers(self, expectation_line):
 146         if self._is_lint_mode:
 147             self._lint_line(expectation_line)
 148
 149         parsed_specifiers = set([specifier.lower() for specifier in expectation_line.specifiers])
 150         expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
 151
 152     def _lint_line(self, expectation_line):
 153         expectations = [expectation.lower() for expectation in expectation_line.expectations]
 154         if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectations:
 155             expectation_line.warnings.append(self.MISSING_BUG_WARNING)
 156         if self.REBASELINE_MODIFIER in expectations:
 157             expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
 158
 159         if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_REBASELINE_MODIFIER in expectations:
 160             for test in expectation_line.matching_tests:
 161                 if self._port.reference_files(test):
 162                     expectation_line.warnings.append('A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline')
 163
 164     def _parse_expectations(self, expectation_line):
 165         result = set()
 166         for part in expectation_line.expectations:
 167             expectation = TestExpectations.expectation_from_string(part)
 168             if expectation is None:  # Careful, PASS is currently 0.
 169                 expectation_line.warnings.append('Unsupported expectation: %s' % part)
 170                 continue
 171             result.add(expectation)
 172         expectation_line.parsed_expectations = result
 173
 174     def _check_test_exists(self, expectation_line):
 175         # WebKit's way of skipping tests is to add a -disabled suffix.
 176         # So we should consider the path existing if the path or the
 177         # -disabled version exists.
 178         if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
 179             # Log a warning here since you hit this case any
 180             # time you update TestExpectations without syncing
 181             # the LayoutTests directory
 182             expectation_line.warnings.append('Path does not exist.')
 183             return False
 184         return True
 185
 186     def _collect_matching_tests(self, expectation_line):
 187         """Convert the test specification to an absolute, normalized
 188         path and make sure directories end with the OS path separator."""
 189         # FIXME: full_test_list can quickly contain a big amount of
 190         # elements. We should consider at some point to use a more
 191         # efficient structure instead of a list. Maybe a dictionary of
 192         # lists to represent the tree of tests, leaves being test
 193         # files and nodes being categories.
 194
 195         if not self._full_test_list:
 196             expectation_line.matching_tests = [expectation_line.path]
 197             return
 198
 199         if not expectation_line.is_file:
 200             # this is a test category, return all the tests of the category.
 201             expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
 202             return
 203
 204         # this is a test file, do a quick check if it's in the
 205         # full test suite.
 206         if expectation_line.path in self._full_test_list:
 207             expectation_line.matching_tests.append(expectation_line.path)
 208
 209     # FIXME: Update the original specifiers and remove this once the old syntax is gone.
 210     _configuration_tokens_list = [
 211         'Mac', 'SnowLeopard', 'Lion', 'Retina', 'MountainLion', 'Mavericks',
 212         'Win', 'XP', 'Win7',
 213         'Linux',
 214         'Android',
 215         'Release',
 216         'Debug',
 217     ]
 218
 219     _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
 220     _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
 221
 222     # FIXME: Update the original specifiers list and remove this once the old syntax is gone.
 223     _expectation_tokens = {
 224         'Crash': 'CRASH',
 225         'Leak': 'LEAK',
 226         'Failure': 'FAIL',
 227         'ImageOnlyFailure': 'IMAGE',
 228         MISSING_KEYWORD: 'MISSING',
 229         'Pass': 'PASS',
 230         'Rebaseline': 'REBASELINE',
 231         NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE',
 232         NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE',
 233         'Skip': 'SKIP',
 234         'Slow': 'SLOW',
 235         'Timeout': 'TIMEOUT',
 236         'WontFix': 'WONTFIX',
 237     }
 238
 239     _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
 240                                         [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
 241
 242     # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
 243     @classmethod
 244     def _tokenize_line(cls, filename, expectation_string, line_number):
 245         """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
 246
 247         The new format for a test expectation line is:
 248
 249         [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
 250
 251         Any errant whitespace is not preserved.
 252
 253         """
 254         expectation_line = TestExpectationLine()
 255         expectation_line.original_string = expectation_string
 256         expectation_line.filename = filename
 257         expectation_line.line_numbers = str(line_number)
 258
 259         comment_index = expectation_string.find("#")
 260         if comment_index == -1:
 261             comment_index = len(expectation_string)
 262         else:
 263             expectation_line.comment = expectation_string[comment_index + 1:]
 264
 265         remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
 266         if len(remaining_string) == 0:
 267             return expectation_line
 268
 269         # special-case parsing this so that we fail immediately instead of treating this as a test name
 270         if remaining_string.startswith('//'):
 271             expectation_line.warnings = ['use "#" instead of "//" for comments']
 272             return expectation_line
 273
 274         bugs = []
 275         specifiers = []
 276         name = None
 277         expectations = []
 278         warnings = []
 279         has_unrecognized_expectation = False
 280
 281         tokens = remaining_string.split()
 282         state = 'start'
 283         for token in tokens:
 284             if (token.startswith(WEBKIT_BUG_PREFIX) or
 285                 token.startswith(CHROMIUM_BUG_PREFIX) or
 286                 token.startswith(V8_BUG_PREFIX) or
 287                 token.startswith(NAMED_BUG_PREFIX)):
 288                 if state != 'start':
 289                     warnings.append('"%s" is not at the start of the line.' % token)
 290                     break
 291                 if token.startswith(WEBKIT_BUG_PREFIX):
 292                     bugs.append(token)
 293                 elif token.startswith(CHROMIUM_BUG_PREFIX):
 294                     bugs.append(token)
 295                 elif token.startswith(V8_BUG_PREFIX):
 296                     bugs.append(token)
 297                 else:
 298                     match = re.match('Bug\((\w+)\)$', token)
 299                     if not match:
 300                         warnings.append('unrecognized bug identifier "%s"' % token)
 301                         break
 302                     else:
 303                         bugs.append(token)
 304             elif token == '[':
 305                 if state == 'start':
 306                     state = 'configuration'
 307                 elif state == 'name_found':
 308                     state = 'expectations'
 309                 else:
 310                     warnings.append('unexpected "["')
 311                     break
 312             elif token == ']':
 313                 if state == 'configuration':
 314                     state = 'name'
 315                 elif state == 'expectations':
 316                     state = 'done'
 317                 else:
 318                     warnings.append('unexpected "]"')
 319                     break
 320             elif token in ('//', ':', '='):
 321                 warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
 322                 break
 323             elif state == 'configuration':
 324                 specifiers.append(cls._configuration_tokens.get(token, token))
 325             elif state == 'expectations':
 326                 if token not in cls._expectation_tokens:
 327                     has_unrecognized_expectation = True
 328                     warnings.append('Unrecognized expectation "%s"' % token)
 329                 else:
 330                     expectations.append(cls._expectation_tokens.get(token, token))
 331             elif state == 'name_found':
 332                 warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
 333                 break
 334             else:
 335                 name = token
 336                 state = 'name_found'
 337
 338         if not warnings:
 339             if not name:
 340                 warnings.append('Did not find a test name.')
 341             elif state not in ('name_found', 'done'):
 342                 warnings.append('Missing a "]"')
 343
 344         if 'WONTFIX' in expectations and 'SKIP' not in expectations:
 345             expectations.append('SKIP')
 346
 347         if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(expectations) - set(['SKIP', 'WONTFIX'])):
 348             warnings.append('A test marked Skip or WontFix must not have other expectations.')
 349
 350         if not expectations and not has_unrecognized_expectation:
 351             warnings.append('Missing expectations.')
 352
 353         expectation_line.bugs = bugs
 354         expectation_line.specifiers = specifiers
 355         expectation_line.expectations = expectations
 356         expectation_line.name = name
 357         expectation_line.warnings = warnings
 358         return expectation_line
 359
 360     @classmethod
 361     def _split_space_separated(cls, space_separated_string):
 362         """Splits a space-separated string into an array."""
 363         return [part.strip() for part in space_separated_string.strip().split(' ')]
 364
 365
 366 class TestExpectationLine(object):
 367     """Represents a line in test expectations file."""
 368
 369     def __init__(self):
 370         """Initializes a blank-line equivalent of an expectation."""
 371         self.original_string = None
 372         self.filename = None  # this is the path to the expectations file for this line
 373         self.line_numbers = "0"
 374         self.name = None  # this is the path in the line itself
 375         self.path = None  # this is the normpath of self.name
 376         self.bugs = []
 377         self.specifiers = []
 378         self.parsed_specifiers = []
 379         self.matching_configurations = set()
 380         self.expectations = []
 381         self.parsed_expectations = set()
 382         self.comment = None
 383         self.matching_tests = []
 384         self.warnings = []
 385         self.is_skipped_outside_expectations_file = False
 386
 387     def __eq__(self, other):
 388         return (self.original_string == other.original_string
 389             and self.filename == other.filename
 390             and self.line_numbers == other.line_numbers
 391             and self.name == other.name
 392             and self.path == other.path
 393             and self.bugs == other.bugs
 394             and self.specifiers == other.specifiers
 395             and self.parsed_specifiers == other.parsed_specifiers
 396             and self.matching_configurations == other.matching_configurations
 397             and self.expectations == other.expectations
 398             and self.parsed_expectations == other.parsed_expectations
 399             and self.comment == other.comment
 400             and self.matching_tests == other.matching_tests
 401             and self.warnings == other.warnings
 402             and self.is_skipped_outside_expectations_file == other.is_skipped_outside_expectations_file)
 403
 404     def is_invalid(self):
 405         return bool(self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING])
 406
 407     def is_flaky(self):
 408         return len(self.parsed_expectations) > 1
 409
 410     def is_whitespace_or_comment(self):
 411         return bool(re.match("^\s*$", self.original_string.split('#')[0]))
 412
 413     @staticmethod
 414     def create_passing_expectation(test):
 415         expectation_line = TestExpectationLine()
 416         expectation_line.name = test
 417         expectation_line.path = test
 418         expectation_line.parsed_expectations = set([PASS])
 419         expectation_line.expectations = set(['PASS'])
 420         expectation_line.matching_tests = [test]
 421         return expectation_line
 422
 423     @staticmethod
 424     def merge_expectation_lines(line1, line2, model_all_expectations):
 425         """Merges the expectations of line2 into line1 and returns a fresh object."""
 426         if line1 is None:
 427             return line2
 428         if line2 is None:
 429             return line1
 430         if model_all_expectations and line1.filename != line2.filename:
 431             return line2
 432
 433         # Don't merge original_string or comment.
 434         result = TestExpectationLine()
 435         # We only care about filenames when we're linting, in which case the filenames are the same.
 436         # Not clear that there's anything better to do when not linting and the filenames are different.
 437         if model_all_expectations:
 438             result.filename = line2.filename
 439         result.line_numbers = line1.line_numbers + "," + line2.line_numbers
 440         result.name = line1.name
 441         result.path = line1.path
 442         result.parsed_expectations = set(line1.parsed_expectations) | set(line2.parsed_expectations)
 443         result.expectations = list(set(line1.expectations) | set(line2.expectations))
 444         result.bugs = list(set(line1.bugs) | set(line2.bugs))
 445         result.specifiers = list(set(line1.specifiers) | set(line2.specifiers))
 446         result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2.parsed_specifiers))
 447         result.matching_configurations = set(line1.matching_configurations) | set(line2.matching_configurations)
 448         result.matching_tests = list(list(set(line1.matching_tests) | set(line2.matching_tests)))
 449         result.warnings = list(set(line1.warnings) | set(line2.warnings))
 450         result.is_skipped_outside_expectations_file = line1.is_skipped_outside_expectations_file or line2.is_skipped_outside_expectations_file
 451         return result
 452
 453     def to_string(self, test_configuration_converter, include_specifiers=True, include_expectations=True, include_comment=True):
 454         parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
 455
 456         if self.is_invalid():
 457             return self.original_string or ''
 458
 459         if self.name is None:
 460             return '' if self.comment is None else "#%s" % self.comment
 461
 462         if test_configuration_converter and self.bugs:
 463             specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations)
 464             result = []
 465             for specifiers in specifiers_list:
 466                 # FIXME: this is silly that we join the specifiers and then immediately split them.
 467                 specifiers = self._serialize_parsed_specifiers(test_configuration_converter, specifiers).split()
 468                 expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split()
 469                 result.append(self._format_line(self.bugs, specifiers, self.name, expectations, self.comment))
 470             return "\n".join(result) if result else None
 471
 472         return self._format_line(self.bugs, self.specifiers, self.name, self.expectations, self.comment,
 473             include_specifiers, include_expectations, include_comment)
 474
 475     def to_csv(self):
 476         # Note that this doesn't include the comments.
 477         return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.specifiers), ' '.join(self.expectations))
 478
 479     def _serialize_parsed_expectations(self, parsed_expectation_to_string):
 480         result = []
 481         for index in TestExpectations.EXPECTATIONS.values():
 482             if index in self.parsed_expectations:
 483                 result.append(parsed_expectation_to_string[index])
 484         return ' '.join(result)
 485
 486     def _serialize_parsed_specifiers(self, test_configuration_converter, specifiers):
 487         result = []
 488         result.extend(sorted(self.parsed_specifiers))
 489         result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers))
 490         return ' '.join(result)
 491
 492     @staticmethod
 493     def _filter_redundant_expectations(expectations):
 494         if set(expectations) == set(['Pass', 'Skip']):
 495             return ['Skip']
 496         if set(expectations) == set(['Pass', 'Slow']):
 497             return ['Slow']
 498         return expectations
 499
 500     @staticmethod
 501     def _format_line(bugs, specifiers, name, expectations, comment, include_specifiers=True, include_expectations=True, include_comment=True):
 502         new_specifiers = []
 503         new_expectations = []
 504         for specifier in specifiers:
 505             # FIXME: Make this all work with the mixed-cased specifiers (e.g. WontFix, Slow, etc).
 506             specifier = specifier.upper()
 507             new_specifiers.append(TestExpectationParser._inverted_configuration_tokens.get(specifier, specifier))
 508
 509         for expectation in expectations:
 510             expectation = expectation.upper()
 511             new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation))
 512
 513         result = ''
 514         if include_specifiers and (bugs or new_specifiers):
 515             if bugs:
 516                 result += ' '.join(bugs) + ' '
 517             if new_specifiers:
 518                 result += '[ %s ] ' % ' '.join(new_specifiers)
 519         result += name
 520         if include_expectations and new_expectations:
 521             new_expectations = TestExpectationLine._filter_redundant_expectations(new_expectations)
 522             result += ' [ %s ]' % ' '.join(sorted(set(new_expectations)))
 523         if include_comment and comment is not None:
 524             result += " #%s" % comment
 525         return result
 526
 527
 528 # FIXME: Refactor API to be a proper CRUD.
 529 class TestExpectationsModel(object):
 530     """Represents relational store of all expectations and provides CRUD semantics to manage it."""
 531
 532     def __init__(self, shorten_filename=None):
 533         # Maps a test to its list of expectations.
 534         self._test_to_expectations = {}
 535
 536         # Maps a test to list of its specifiers (string values)
 537         self._test_to_specifiers = {}
 538
 539         # Maps a test to a TestExpectationLine instance.
 540         self._test_to_expectation_line = {}
 541
 542         self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS)
 543         self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES)
 544         self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES)
 545
 546         self._shorten_filename = shorten_filename or (lambda x: x)
 547
 548     def _merge_test_map(self, self_map, other_map):
 549         for test in other_map:
 550             new_expectations = set(other_map[test])
 551             if test in self_map:
 552                 new_expectations |= set(self_map[test])
 553             self_map[test] = list(new_expectations) if isinstance(other_map[test], list) else new_expectations
 554
 555     def _merge_dict_of_sets(self, self_dict, other_dict):
 556         for key in other_dict:
 557             self_dict[key] |= other_dict[key]
 558
 559     def merge_model(self, other):
 560         self._merge_test_map(self._test_to_expectations, other._test_to_expectations)
 561
 562         for test, line in other._test_to_expectation_line.items():
 563             if test in self._test_to_expectation_line:
 564                 line = TestExpectationLine.merge_expectation_lines(self._test_to_expectation_line[test], line, model_all_expectations=False)
 565             self._test_to_expectation_line[test] = line
 566
 567         self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_to_tests)
 568         self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tests)
 569         self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_to_tests)
 570
 571     def _dict_of_sets(self, strings_to_constants):
 572         """Takes a dict of strings->constants and returns a dict mapping
 573         each constant to an empty set."""
 574         d = {}
 575         for c in strings_to_constants.values():
 576             d[c] = set()
 577         return d
 578
 579     def get_test_set(self, expectation, include_skips=True):
 580         tests = self._expectation_to_tests[expectation]
 581         if not include_skips:
 582             tests = tests - self.get_test_set(SKIP)
 583         return tests
 584
 585     def get_test_set_for_keyword(self, keyword):
 586         expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None)
 587         if expectation_enum is not None:
 588             return self._expectation_to_tests[expectation_enum]
 589
 590         matching_tests = set()
 591         for test, specifiers in self._test_to_specifiers.iteritems():
 592             if keyword.lower() in specifiers:
 593                 matching_tests.add(test)
 594         return matching_tests
 595
 596     def get_tests_with_result_type(self, result_type):
 597         return self._result_type_to_tests[result_type]
 598
 599     def get_tests_with_timeline(self, timeline):
 600         return self._timeline_to_tests[timeline]
 601
 602     def has_test(self, test):
 603         return test in self._test_to_expectation_line
 604
 605     def get_expectation_line(self, test):
 606         return self._test_to_expectation_line.get(test)
 607
 608     def get_expectations(self, test):
 609         return self._test_to_expectations[test]
 610
 611     def get_expectations_string(self, test):
 612         """Returns the expectatons for the given test as an uppercase string.
 613         If there are no expectations for the test, then "PASS" is returned."""
 614         if self.get_expectation_line(test).is_skipped_outside_expectations_file:
 615             return 'NOTRUN'
 616
 617         expectations = self.get_expectations(test)
 618         retval = []
 619
 620         # FIXME: WontFix should cause the test to get skipped without artificially adding SKIP to the expectations list.
 621         if WONTFIX in expectations and SKIP in expectations:
 622             expectations.remove(SKIP)
 623
 624         for expectation in expectations:
 625             retval.append(self.expectation_to_string(expectation))
 626
 627         return " ".join(retval)
 628
 629     def expectation_to_string(self, expectation):
 630         """Return the uppercased string equivalent of a given expectation."""
 631         for item in TestExpectations.EXPECTATIONS.items():
 632             if item[1] == expectation:
 633                 return item[0].upper()
 634         raise ValueError(expectation)
 635
 636     def remove_expectation_line(self, test):
 637         if not self.has_test(test):
 638             return
 639         self._clear_expectations_for_test(test)
 640         del self._test_to_expectation_line[test]
 641
 642     def add_expectation_line(self, expectation_line,
 643                              model_all_expectations=False):
 644         """Returns a list of warnings encountered while matching specifiers."""
 645
 646         if expectation_line.is_invalid():
 647             return
 648
 649         for test in expectation_line.matching_tests:
 650             if self._already_seen_better_match(test, expectation_line):
 651                 continue
 652
 653             if model_all_expectations:
 654                 expectation_line = TestExpectationLine.merge_expectation_lines(self.get_expectation_line(test), expectation_line, model_all_expectations)
 655
 656             self._clear_expectations_for_test(test)
 657             self._test_to_expectation_line[test] = expectation_line
 658             self._add_test(test, expectation_line)
 659
 660     def _add_test(self, test, expectation_line):
 661         """Sets the expected state for a given test.
 662
 663         This routine assumes the test has not been added before. If it has,
 664         use _clear_expectations_for_test() to reset the state prior to
 665         calling this."""
 666         self._test_to_expectations[test] = expectation_line.parsed_expectations
 667         for expectation in expectation_line.parsed_expectations:
 668             self._expectation_to_tests[expectation].add(test)
 669
 670         self._test_to_specifiers[test] = expectation_line.specifiers
 671
 672         if WONTFIX in expectation_line.parsed_expectations:
 673             self._timeline_to_tests[WONTFIX].add(test)
 674         else:
 675             self._timeline_to_tests[NOW].add(test)
 676
 677         if SKIP in expectation_line.parsed_expectations:
 678             self._result_type_to_tests[SKIP].add(test)
 679         elif expectation_line.parsed_expectations == set([PASS]):
 680             self._result_type_to_tests[PASS].add(test)
 681         elif expectation_line.is_flaky():
 682             self._result_type_to_tests[FLAKY].add(test)
 683         else:
 684             # FIXME: What is this?
 685             self._result_type_to_tests[FAIL].add(test)
 686
 687     def _clear_expectations_for_test(self, test):
 688         """Remove prexisting expectations for this test.
 689         This happens if we are seeing a more precise path
 690         than a previous listing.
 691         """
 692         if self.has_test(test):
 693             self._test_to_expectations.pop(test, '')
 694             self._remove_from_sets(test, self._expectation_to_tests)
 695             self._remove_from_sets(test, self._timeline_to_tests)
 696             self._remove_from_sets(test, self._result_type_to_tests)
 697
 698     def _remove_from_sets(self, test, dict_of_sets_of_tests):
 699         """Removes the given test from the sets in the dictionary.
 700
 701         Args:
 702           test: test to look for
 703           dict: dict of sets of files"""
 704         for set_of_tests in dict_of_sets_of_tests.itervalues():
 705             if test in set_of_tests:
 706                 set_of_tests.remove(test)
 707
 708     def _already_seen_better_match(self, test, expectation_line):
 709         """Returns whether we've seen a better match already in the file.
 710
 711         Returns True if we've already seen a expectation_line.name that matches more of the test
 712             than this path does
 713         """
 714         # FIXME: See comment below about matching test configs and specificity.
 715         if not self.has_test(test):
 716             # We've never seen this test before.
 717             return False
 718
 719         prev_expectation_line = self._test_to_expectation_line[test]
 720
 721         if prev_expectation_line.filename != expectation_line.filename:
 722             # We've moved on to a new expectation file, which overrides older ones.
 723             return False
 724
 725         if len(prev_expectation_line.path) > len(expectation_line.path):
 726             # The previous path matched more of the test.
 727             return True
 728
 729         if len(prev_expectation_line.path) < len(expectation_line.path):
 730             # This path matches more of the test.
 731             return False
 732
 733         # At this point we know we have seen a previous exact match on this
 734         # base path, so we need to check the two sets of specifiers.
 735
 736         # FIXME: This code was originally designed to allow lines that matched
 737         # more specifiers to override lines that matched fewer specifiers.
 738         # However, we currently view these as errors.
 739         #
 740         # To use the "more specifiers wins" policy, change the errors for overrides
 741         # to be warnings and return False".
 742
 743         if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
 744             expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%s and %s:%s.' % (
 745                 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers,
 746                 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers))
 747             return True
 748
 749         if prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
 750             expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name,
 751                 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers,
 752                 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers))
 753             # FIXME: return False if we want more specific to win.
 754             return True
 755
 756         if prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
 757             expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name,
 758                 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers,
 759                 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers))
 760             return True
 761
 762         if prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
 763             expectation_line.warnings.append('Entries for %s on lines %s:%s and %s:%s match overlapping sets of configurations.' % (expectation_line.name,
 764                 self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers,
 765                 self._shorten_filename(expectation_line.filename), expectation_line.line_numbers))
 766             return True
 767
 768         # Configuration sets are disjoint, then.
 769         return False
 770
 771
 772 class TestExpectations(object):
 773     """Test expectations consist of lines with specifications of what
 774     to expect from layout test cases. The test cases can be directories
 775     in which case the expectations apply to all test cases in that
 776     directory and any subdirectory. The format is along the lines of:
 777
 778       LayoutTests/fast/js/fixme.js [ Failure ]
 779       LayoutTests/fast/js/flaky.js [ Failure Pass ]
 780       LayoutTests/fast/js/crash.js [ Crash Failure Pass Timeout ]
 781       ...
 782
 783     To add specifiers:
 784       LayoutTests/fast/js/no-good.js
 785       [ Debug ] LayoutTests/fast/js/no-good.js [ Pass Timeout ]
 786       [ Debug ] LayoutTests/fast/js/no-good.js [ Pass Skip Timeout ]
 787       [ Linux Debug ] LayoutTests/fast/js/no-good.js [ Pass Skip Timeout ]
 788       [ Linux Win ] LayoutTests/fast/js/no-good.js [ Pass Skip Timeout ]
 789
 790     Skip: Doesn't run the test.
 791     Slow: The test takes a long time to run, but does not timeout indefinitely.
 792     WontFix: For tests that we never intend to pass on a given platform (treated like Skip).
 793
 794     Notes:
 795       -A test cannot be both SLOW and TIMEOUT
 796       -A test can be included twice, but not via the same path.
 797       -If a test is included twice, then the more precise path wins.
 798       -CRASH tests cannot be WONTFIX
 799     """
 800
 801     # FIXME: Update to new syntax once the old format is no longer supported.
 802     EXPECTATIONS = {'pass': PASS,
 803                     'audio': AUDIO,
 804                     'fail': FAIL,
 805                     'image': IMAGE,
 806                     'image+text': IMAGE_PLUS_TEXT,
 807                     'text': TEXT,
 808                     'timeout': TIMEOUT,
 809                     'crash': CRASH,
 810                     'leak': LEAK,
 811                     'missing': MISSING,
 812                     TestExpectationParser.SKIP_MODIFIER: SKIP,
 813                     TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBASELINE,
 814                     TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEEDS_MANUAL_REBASELINE,
 815                     TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
 816                     TestExpectationParser.SLOW_MODIFIER: SLOW,
 817                     TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,
 818     }
 819
 820     EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems())
 821
 822     # (aggregated by category, pass/fail/skip, type)
 823     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
 824                                 PASS: 'passes',
 825                                 FAIL: 'failures',
 826                                 IMAGE: 'image-only failures',
 827                                 TEXT: 'text-only failures',
 828                                 IMAGE_PLUS_TEXT: 'image and text failures',
 829                                 AUDIO: 'audio failures',
 830                                 CRASH: 'crashes',
 831                                 LEAK: 'leaks',
 832                                 TIMEOUT: 'timeouts',
 833                                 MISSING: 'missing results'}
 834
 835     NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX)
 836
 837     BUILD_TYPES = ('debug', 'release')
 838
 839     TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
 840                  'now': NOW}
 841
 842     RESULT_TYPES = {'skip': SKIP,
 843                     'pass': PASS,
 844                     'fail': FAIL,
 845                     'flaky': FLAKY}
 846
 847     @classmethod
 848     def expectation_from_string(cls, string):
 849         assert(' ' not in string)  # This only handles one expectation at a time.
 850         return cls.EXPECTATIONS.get(string.lower())
 851
 852     @staticmethod
 853     def result_was_expected(result, expected_results, test_needs_rebaselining):
 854         """Returns whether we got a result we were expecting.
 855         Args:
 856             result: actual result of a test execution
 857             expected_results: set of results listed in test_expectations
 858             test_needs_rebaselining: whether test was marked as REBASELINE"""
 859         if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_EXPECTATIONS))):
 860             expected_results = set([PASS])
 861
 862         if result in expected_results:
 863             return True
 864         if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE in expected_results):
 865             return True
 866         if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
 867             return True
 868         if result == MISSING and test_needs_rebaselining:
 869             return True
 870         if result == SKIP:
 871             return True
 872         return False
 873
 874     @staticmethod
 875     def remove_pixel_failures(expected_results):
 876         """Returns a copy of the expected results for a test, except that we
 877         drop any pixel failures and return the remaining expectations. For example,
 878         if we're not running pixel tests, then tests expected to fail as IMAGE
 879         will PASS."""
 880         expected_results = expected_results.copy()
 881         if IMAGE in expected_results:
 882             expected_results.remove(IMAGE)
 883             expected_results.add(PASS)
 884         return expected_results
 885
 886     @staticmethod
 887     def remove_non_sanitizer_failures(expected_results):
 888         """Returns a copy of the expected results for a test, except that we
 889         drop any failures that the sanitizers don't care about."""
 890         expected_results = expected_results.copy()
 891         for result in (IMAGE, FAIL, IMAGE_PLUS_TEXT):
 892             if result in expected_results:
 893                 expected_results.remove(result)
 894                 expected_results.add(PASS)
 895         return expected_results
 896
 897     @staticmethod
 898     def has_pixel_failures(actual_results):
 899         return IMAGE in actual_results or FAIL in actual_results
 900
 901     @staticmethod
 902     def suffixes_for_expectations(expectations):
 903         suffixes = set()
 904         if IMAGE in expectations:
 905             suffixes.add('png')
 906         if FAIL in expectations:
 907             suffixes.add('txt')
 908             suffixes.add('png')
 909             suffixes.add('wav')
 910         return set(suffixes)
 911
 912     @staticmethod
 913     def suffixes_for_actual_expectations_string(expectations):
 914         suffixes = set()
 915         if 'TEXT' in expectations:
 916             suffixes.add('txt')
 917         if 'IMAGE' in expectations:
 918             suffixes.add('png')
 919         if 'AUDIO' in expectations:
 920             suffixes.add('wav')
 921         if 'MISSING' in expectations:
 922             suffixes.add('txt')
 923             suffixes.add('png')
 924             suffixes.add('wav')
 925         return suffixes
 926
 927     # FIXME: This constructor does too much work. We should move the actual parsing of
 928     # the expectations into separate routines so that linting and handling overrides
 929     # can be controlled separately, and the constructor can be more of a no-op.
 930     def __init__(self, port, tests=None, include_overrides=True, expectations_dict=None, model_all_expectations=False, is_lint_mode=False):
 931         self._full_test_list = tests
 932         self._test_config = port.test_configuration()
 933         self._is_lint_mode = is_lint_mode
 934         self._model_all_expectations = self._is_lint_mode or model_all_expectations
 935         self._model = TestExpectationsModel(self._shorten_filename)
 936         self._parser = TestExpectationParser(port, tests, self._is_lint_mode)
 937         self._port = port
 938         self._skipped_tests_warnings = []
 939         self._expectations = []
 940
 941         if not expectations_dict:
 942             expectations_dict = port.expectations_dict()
 943
 944         # Always parse the generic expectations (the generic file is required
 945         # to be the first one in the expectations_dict, which must be an OrderedDict).
 946         generic_path, generic_exps = expectations_dict.items()[0]
 947         expectations = self._parser.parse(generic_path, generic_exps)
 948         self._add_expectations(expectations, self._model)
 949         self._expectations += expectations
 950
 951         # Now add the overrides if so requested.
 952         if include_overrides:
 953             for path, contents in expectations_dict.items()[1:]:
 954                 expectations = self._parser.parse(path, contents)
 955                 model = TestExpectationsModel(self._shorten_filename)
 956                 self._add_expectations(expectations, model)
 957                 self._expectations += expectations
 958                 self._model.merge_model(model)
 959
 960         # FIXME: move ignore_tests into port.skipped_layout_tests()
 961         self.add_extra_skipped_tests(port.skipped_layout_tests(tests).union(set(port.get_option('ignore_tests', []))))
 962         self.add_expectations_from_bot()
 963
 964         self._has_warnings = False
 965         self._report_warnings()
 966         self._process_tests_without_expectations()
 967
 968     # TODO(ojan): Allow for removing skipped tests when getting the list of
 969     # tests to run, but not when getting metrics.
 970     def model(self):
 971         return self._model
 972
 973     def get_needs_rebaseline_failures(self):
 974         return self._model.get_test_set(NEEDS_REBASELINE)
 975
 976     def get_rebaselining_failures(self):
 977         return self._model.get_test_set(REBASELINE)
 978
 979     # FIXME: Change the callsites to use TestExpectationsModel and remove.
 980     def get_expectations(self, test):
 981         return self._model.get_expectations(test)
 982
 983     # FIXME: Change the callsites to use TestExpectationsModel and remove.
 984     def get_tests_with_result_type(self, result_type):
 985         return self._model.get_tests_with_result_type(result_type)
 986
 987     # FIXME: Change the callsites to use TestExpectationsModel and remove.
 988     def get_test_set(self, expectation, include_skips=True):
 989         return self._model.get_test_set(expectation, include_skips)
 990
 991     # FIXME: Change the callsites to use TestExpectationsModel and remove.
 992     def get_tests_with_timeline(self, timeline):
 993         return self._model.get_tests_with_timeline(timeline)
 994
 995     def get_expectations_string(self, test):
 996         return self._model.get_expectations_string(test)
 997
 998     def expectation_to_string(self, expectation):
 999         return self._model.expectation_to_string(expectation)
1000
1001     def matches_an_expected_result(self, test, result, pixel_tests_are_enabled, sanitizer_is_enabled):
1002         expected_results = self._model.get_expectations(test)
1003         if sanitizer_is_enabled:
1004             expected_results = self.remove_non_sanitizer_failures(expected_results)
1005         elif not pixel_tests_are_enabled:
1006             expected_results = self.remove_pixel_failures(expected_results)
1007         return self.result_was_expected(result, expected_results, self.is_rebaselining(test))
1008
1009     def is_rebaselining(self, test):
1010         return REBASELINE in self._model.get_expectations(test)
1011
1012     def _shorten_filename(self, filename):
1013         if filename.startswith(self._port.path_from_webkit_base()):
1014             return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base())
1015         return filename
1016
1017     def _report_warnings(self):
1018         warnings = []
1019         for expectation in self._expectations:
1020             for warning in expectation.warnings:
1021                 warnings.append('%s:%s %s %s' % (self._shorten_filename(expectation.filename), expectation.line_numbers,
1022                                 warning, expectation.name if expectation.expectations else expectation.original_string))
1023
1024         if warnings:
1025             self._has_warnings = True
1026             if self._is_lint_mode:
1027                 raise ParseError(warnings)
1028             _log.warning('--lint-test-files warnings:')
1029             for warning in warnings:
1030                 _log.warning(warning)
1031             _log.warning('')
1032
1033     def _process_tests_without_expectations(self):
1034         if self._full_test_list:
1035             for test in self._full_test_list:
1036                 if not self._model.has_test(test):
1037                     self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test))
1038
1039     def has_warnings(self):
1040         return self._has_warnings
1041
1042     def remove_configurations(self, removals):
1043         expectations_to_remove = []
1044         modified_expectations = []
1045
1046         for test, test_configuration in removals:
1047             for expectation in self._expectations:
1048                 if expectation.name != test or not expectation.parsed_expectations:
1049                     continue
1050                 if test_configuration not in expectation.matching_configurations:
1051                     continue
1052
1053                 expectation.matching_configurations.remove(test_configuration)
1054                 if expectation.matching_configurations:
1055                     modified_expectations.append(expectation)
1056                 else:
1057                     expectations_to_remove.append(expectation)
1058
1059         for expectation in expectations_to_remove:
1060             index = self._expectations.index(expectation)
1061             self._expectations.remove(expectation)
1062
1063             if index == len(self._expectations) or self._expectations[index].is_whitespace_or_comment():
1064                 while index and self._expectations[index - 1].is_whitespace_or_comment():
1065                     index = index - 1
1066                     self._expectations.pop(index)
1067
1068         return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations)
1069
1070     def _add_expectations(self, expectation_list, model):
1071         for expectation_line in expectation_list:
1072             if not expectation_line.expectations:
1073                 continue
1074
1075             if self._model_all_expectations or self._test_config in expectation_line.matching_configurations:
1076                 model.add_expectation_line(expectation_line, model_all_expectations=self._model_all_expectations)
1077
1078     def add_extra_skipped_tests(self, tests_to_skip):
1079         if not tests_to_skip:
1080             return
1081         for test in self._expectations:
1082             if test.name and test.name in tests_to_skip:
1083                 test.warnings.append('%s:%s %s is also in a Skipped file.' % (test.filename, test.line_numbers, test.name))
1084
1085         model = TestExpectationsModel(self._shorten_filename)
1086         for test_name in tests_to_skip:
1087             expectation_line = self._parser.expectation_for_skipped_test(test_name)
1088             model.add_expectation_line(expectation_line)
1089         self._model.merge_model(model)
1090
1091     def add_expectations_from_bot(self):
1092         # FIXME: With mode 'very-flaky' and 'maybe-flaky', this will show the expectations entry in the flakiness
1093         # dashboard rows for each test to be whatever the bot thinks they should be. Is this a good thing?
1094         bot_expectations = self._port.bot_expectations()
1095         model = TestExpectationsModel(self._shorten_filename)
1096         for test_name in bot_expectations:
1097             expectation_line = self._parser.expectation_line_for_test(test_name, bot_expectations[test_name])
1098
1099             # Unexpected results are merged into existing expectations.
1100             merge = self._port.get_option('ignore_flaky_tests') == 'unexpected'
1101             model.add_expectation_line(expectation_line)
1102         self._model.merge_model(model)
1103
1104     def add_expectation_line(self, expectation_line):
1105         self._model.add_expectation_line(expectation_line)
1106         self._expectations += [expectation_line]
1107
1108     def remove_expectation_line(self, test):
1109         if not self._model.has_test(test):
1110             return
1111         self._expectations.remove(self._model.get_expectation_line(test))
1112         self._model.remove_expectation_line(test)
1113
1114     @staticmethod
1115     def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None):
1116         def serialize(expectation_line):
1117             # If reconstitute_only_these is an empty list, we want to return original_string.
1118             # So we need to compare reconstitute_only_these to None, not just check if it's falsey.
1119             if reconstitute_only_these is None or expectation_line in reconstitute_only_these:
1120                 return expectation_line.to_string(test_configuration_converter)
1121             return expectation_line.original_string
1122
1123         def nones_out(expectation_line):
1124             return expectation_line is not None
1125
1126         return "\n".join(filter(nones_out, map(serialize, expectation_lines)))