src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the Google name nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 """Abstract base class of Port-specific entry points for the layout tests
  30 test infrastructure (the Port and Driver classes)."""
  31
  32 import cgi
  33 import difflib
  34 import errno
  35 import itertools
  36 import logging
  37 import os
  38 import operator
  39 import optparse
  40 import re
  41 import sys
  42
  43 try:
  44     from collections import OrderedDict
  45 except ImportError:
  46     # Needed for Python < 2.7
  47     from webkitpy.thirdparty.ordered_dict import OrderedDict
  48
  49
  50 from webkitpy.common import find_files
  51 from webkitpy.common import read_checksum_from_png
  52 from webkitpy.common.memoized import memoized
  53 from webkitpy.common.system import path
  54 from webkitpy.common.system.executive import ScriptError
  55 from webkitpy.common.system.path import cygpath
  56 from webkitpy.common.system.systemhost import SystemHost
  57 from webkitpy.common.webkit_finder import WebKitFinder
  58 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
  59 from webkitpy.layout_tests.models import test_run_results
  60 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  61 from webkitpy.layout_tests.port import config as port_config
  62 from webkitpy.layout_tests.port import driver
  63 from webkitpy.layout_tests.port import server_process
  64 from webkitpy.layout_tests.port.factory import PortFactory
  65 from webkitpy.layout_tests.servers import apache_http
  66 from webkitpy.layout_tests.servers import pywebsocket
  67
  68 _log = logging.getLogger(__name__)
  69
  70
  71 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports.
  72 class Port(object):
  73     """Abstract class for Port-specific hooks for the layout_test package."""
  74
  75     # Subclasses override this. This should indicate the basic implementation
  76     # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?)
  77     # one unique value per class.
  78
  79     # FIXME: We should probably rename this to something like 'implementation_name'.
  80     port_name = None
  81
  82     # Test names resemble unix relative paths, and use '/' as a directory separator.
  83     TEST_PATH_SEPARATOR = '/'
  84
  85     ALL_BUILD_TYPES = ('debug', 'release')
  86
  87     CONTENT_SHELL_NAME = 'content_shell'
  88
  89     # True if the port as aac and mp3 codecs built in.
  90     PORT_HAS_AUDIO_CODECS_BUILT_IN = False
  91
  92     ALL_SYSTEMS = (
  93         ('snowleopard', 'x86'),
  94         ('lion', 'x86'),
  95
  96         # FIXME: We treat Retina (High-DPI) devices as if they are running
  97         # a different operating system version. This isn't accurate, but will work until
  98         # we need to test and support baselines across multiple O/S versions.
  99         ('retina', 'x86'),
 100
 101         ('mountainlion', 'x86'),
 102         ('mavericks', 'x86'),
 103         ('xp', 'x86'),
 104         ('win7', 'x86'),
 105         ('lucid', 'x86'),
 106         ('lucid', 'x86_64'),
 107         # FIXME: Technically this should be 'arm', but adding a third architecture type breaks TestConfigurationConverter.
 108         # If we need this to be 'arm' in the future, then we first have to fix TestConfigurationConverter.
 109         ('icecreamsandwich', 'x86'),
 110         )
 111
 112     ALL_BASELINE_VARIANTS = [
 113         'mac-mavericks', 'mac-mountainlion', 'mac-retina', 'mac-lion', 'mac-snowleopard',
 114         'win-win7', 'win-xp',
 115         'linux-x86_64', 'linux-x86',
 116     ]
 117
 118     CONFIGURATION_SPECIFIER_MACROS = {
 119         'mac': ['snowleopard', 'lion', 'retina', 'mountainlion', 'mavericks'],
 120         'win': ['xp', 'win7'],
 121         'linux': ['lucid'],
 122         'android': ['icecreamsandwich'],
 123     }
 124
 125     DEFAULT_BUILD_DIRECTORIES = ('out',)
 126
 127     # overridden in subclasses.
 128     FALLBACK_PATHS = {}
 129
 130     SUPPORTED_VERSIONS = []
 131
 132     # URL to the build requirements page.
 133     BUILD_REQUIREMENTS_URL = ''
 134
 135     @classmethod
 136     def latest_platform_fallback_path(cls):
 137         return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]]
 138
 139     @classmethod
 140     def _static_build_path(cls, filesystem, build_directory, chromium_base, configuration, comps):
 141         if build_directory:
 142             return filesystem.join(build_directory, configuration, *comps)
 143
 144         hits = []
 145         for directory in cls.DEFAULT_BUILD_DIRECTORIES:
 146             base_dir = filesystem.join(chromium_base, directory, configuration)
 147             path = filesystem.join(base_dir, *comps)
 148             if filesystem.exists(path):
 149                 hits.append((filesystem.mtime(path), path))
 150
 151         if hits:
 152             hits.sort(reverse=True)
 153             return hits[0][1]  # Return the newest file found.
 154
 155         # We have to default to something, so pick the last one.
 156         return filesystem.join(base_dir, *comps)
 157
 158     @classmethod
 159     def determine_full_port_name(cls, host, options, port_name):
 160         """Return a fully-specified port name that can be used to construct objects."""
 161         # Subclasses will usually override this.
 162         assert port_name.startswith(cls.port_name)
 163         return port_name
 164
 165     def __init__(self, host, port_name, options=None, **kwargs):
 166
 167         # This value may be different from cls.port_name by having version modifiers
 168         # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2').
 169         self._name = port_name
 170
 171         # These are default values that should be overridden in a subclasses.
 172         self._version = ''
 173         self._architecture = 'x86'
 174
 175         # FIXME: Ideally we'd have a package-wide way to get a
 176         # well-formed options object that had all of the necessary
 177         # options defined on it.
 178         self._options = options or optparse.Values()
 179
 180         self.host = host
 181         self._executive = host.executive
 182         self._filesystem = host.filesystem
 183         self._webkit_finder = WebKitFinder(host.filesystem)
 184         self._config = port_config.Config(self._executive, self._filesystem, self.port_name)
 185
 186         self._helper = None
 187         self._http_server = None
 188         self._websocket_server = None
 189         self._image_differ = None
 190         self._server_process_constructor = server_process.ServerProcess  # overridable for testing
 191         self._http_lock = None  # FIXME: Why does this live on the port object?
 192         self._dump_reader = None
 193
 194         # Python's Popen has a bug that causes any pipes opened to a
 195         # process that can't be executed to be leaked.  Since this
 196         # code is specifically designed to tolerate exec failures
 197         # to gracefully handle cases where wdiff is not installed,
 198         # the bug results in a massive file descriptor leak. As a
 199         # workaround, if an exec failure is ever experienced for
 200         # wdiff, assume it's not available.  This will leak one
 201         # file descriptor but that's better than leaking each time
 202         # wdiff would be run.
 203         #
 204         # http://mail.python.org/pipermail/python-list/
 205         #    2008-August/505753.html
 206         # http://bugs.python.org/issue3210
 207         self._wdiff_available = None
 208
 209         # FIXME: prettypatch.py knows this path, why is it copied here?
 210         self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb")
 211         self._pretty_patch_available = None
 212
 213         if not hasattr(options, 'configuration') or not options.configuration:
 214             self.set_option_default('configuration', self.default_configuration())
 215         self._test_configuration = None
 216         self._reftest_list = {}
 217         self._results_directory = None
 218
 219     def buildbot_archives_baselines(self):
 220         return True
 221
 222     def additional_drt_flag(self):
 223         if self.driver_name() == self.CONTENT_SHELL_NAME:
 224             return ['--dump-render-tree']
 225         return []
 226
 227     def supports_per_test_timeout(self):
 228         return False
 229
 230     def default_pixel_tests(self):
 231         return True
 232
 233     def default_smoke_test_only(self):
 234         return False
 235
 236     def default_timeout_ms(self):
 237         timeout_ms = 6 * 1000
 238         if self.get_option('configuration') == 'Debug':
 239             # Debug is usually 2x-3x slower than Release.
 240             return 3 * timeout_ms
 241         return timeout_ms
 242
 243     def driver_stop_timeout(self):
 244         """ Returns the amount of time in seconds to wait before killing the process in driver.stop()."""
 245         # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as
 246         # well (for things like ASAN, Valgrind, etc.)
 247         return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms()
 248
 249     def wdiff_available(self):
 250         if self._wdiff_available is None:
 251             self._wdiff_available = self.check_wdiff(logging=False)
 252         return self._wdiff_available
 253
 254     def pretty_patch_available(self):
 255         if self._pretty_patch_available is None:
 256             self._pretty_patch_available = self.check_pretty_patch(logging=False)
 257         return self._pretty_patch_available
 258
 259     def default_child_processes(self):
 260         """Return the number of drivers to use for this port."""
 261         return self._executive.cpu_count()
 262
 263     def default_max_locked_shards(self):
 264         """Return the number of "locked" shards to run in parallel (like the http tests)."""
 265         max_locked_shards = int(self.default_child_processes()) / 4
 266         if not max_locked_shards:
 267             return 1
 268         return max_locked_shards
 269
 270     def baseline_path(self):
 271         """Return the absolute path to the directory to store new baselines in for this port."""
 272         # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
 273         return self.baseline_version_dir()
 274
 275     def baseline_platform_dir(self):
 276         """Return the absolute path to the default (version-independent) platform-specific results."""
 277         return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
 278
 279     def baseline_version_dir(self):
 280         """Return the absolute path to the platform-and-version-specific results."""
 281         baseline_search_paths = self.baseline_search_path()
 282         return baseline_search_paths[0]
 283
 284     def virtual_baseline_search_path(self, test_name):
 285         suite = self.lookup_virtual_suite(test_name)
 286         if not suite:
 287             return None
 288         return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()]
 289
 290     def baseline_search_path(self):
 291         return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path()
 292
 293     def default_baseline_search_path(self):
 294         """Return a list of absolute paths to directories to search under for
 295         baselines. The directories are searched in order."""
 296         return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version()])
 297
 298     @memoized
 299     def _compare_baseline(self):
 300         factory = PortFactory(self.host)
 301         target_port = self.get_option('compare_port')
 302         if target_port:
 303             return factory.get(target_port).default_baseline_search_path()
 304         return []
 305
 306     def _check_file_exists(self, path_to_file, file_description,
 307                            override_step=None, logging=True):
 308         """Verify the file is present where expected or log an error.
 309
 310         Args:
 311             file_name: The (human friendly) name or description of the file
 312                 you're looking for (e.g., "HTTP Server"). Used for error logging.
 313             override_step: An optional string to be logged if the check fails.
 314             logging: Whether or not log the error messages."""
 315         if not self._filesystem.exists(path_to_file):
 316             if logging:
 317                 _log.error('Unable to find %s' % file_description)
 318                 _log.error('    at %s' % path_to_file)
 319                 if override_step:
 320                     _log.error('    %s' % override_step)
 321                     _log.error('')
 322             return False
 323         return True
 324
 325     def check_build(self, needs_http, printer):
 326         result = True
 327
 328         dump_render_tree_binary_path = self._path_to_driver()
 329         result = self._check_file_exists(dump_render_tree_binary_path,
 330                                          'test driver') and result
 331         if not result and self.get_option('build'):
 332             result = self._check_driver_build_up_to_date(
 333                 self.get_option('configuration'))
 334         else:
 335             _log.error('')
 336
 337         helper_path = self._path_to_helper()
 338         if helper_path:
 339             result = self._check_file_exists(helper_path,
 340                                              'layout test helper') and result
 341
 342         if self.get_option('pixel_tests'):
 343             result = self.check_image_diff(
 344                 'To override, invoke with --no-pixel-tests') and result
 345
 346         # It's okay if pretty patch and wdiff aren't available, but we will at least log messages.
 347         self._pretty_patch_available = self.check_pretty_patch()
 348         self._wdiff_available = self.check_wdiff()
 349
 350         if self._dump_reader:
 351             result = self._dump_reader.check_is_functional() and result
 352
 353         if needs_http:
 354             result = self.check_httpd() and result
 355
 356         return test_run_results.OK_EXIT_STATUS if result else test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
 357
 358     def _check_driver(self):
 359         driver_path = self._path_to_driver()
 360         if not self._filesystem.exists(driver_path):
 361             _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
 362             return False
 363         return True
 364
 365     def _check_port_build(self):
 366         # Ports can override this method to do additional checks.
 367         return True
 368
 369     def check_sys_deps(self, needs_http):
 370         """If the port needs to do some runtime checks to ensure that the
 371         tests can be run successfully, it should override this routine.
 372         This step can be skipped with --nocheck-sys-deps.
 373
 374         Returns whether the system is properly configured."""
 375         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
 376
 377         local_error = ScriptError()
 378
 379         def error_handler(script_error):
 380             local_error.exit_code = script_error.exit_code
 381
 382         output = self._executive.run_command(cmd, error_handler=error_handler)
 383         if local_error.exit_code:
 384             _log.error('System dependencies check failed.')
 385             _log.error('To override, invoke with --nocheck-sys-deps')
 386             _log.error('')
 387             _log.error(output)
 388             if self.BUILD_REQUIREMENTS_URL is not '':
 389                 _log.error('')
 390                 _log.error('For complete build requirements, please see:')
 391                 _log.error(self.BUILD_REQUIREMENTS_URL)
 392             return test_run_results.SYS_DEPS_EXIT_STATUS
 393         return test_run_results.OK_EXIT_STATUS
 394
 395     def check_image_diff(self, override_step=None, logging=True):
 396         """This routine is used to check whether image_diff binary exists."""
 397         image_diff_path = self._path_to_image_diff()
 398         if not self._filesystem.exists(image_diff_path):
 399             _log.error("image_diff was not found at %s" % image_diff_path)
 400             return False
 401         return True
 402
 403     def check_pretty_patch(self, logging=True):
 404         """Checks whether we can use the PrettyPatch ruby script."""
 405         try:
 406             _ = self._executive.run_command(['ruby', '--version'])
 407         except OSError, e:
 408             if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
 409                 if logging:
 410                     _log.warning("Ruby is not installed; can't generate pretty patches.")
 411                     _log.warning('')
 412                 return False
 413
 414         if not self._filesystem.exists(self._pretty_patch_path):
 415             if logging:
 416                 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path)
 417                 _log.warning('')
 418             return False
 419
 420         return True
 421
 422     def check_wdiff(self, logging=True):
 423         if not self._path_to_wdiff():
 424             # Don't need to log here since this is the port choosing not to use wdiff.
 425             return False
 426
 427         try:
 428             _ = self._executive.run_command([self._path_to_wdiff(), '--help'])
 429         except OSError:
 430             if logging:
 431                 message = self._wdiff_missing_message()
 432                 if message:
 433                     for line in message.splitlines():
 434                         _log.warning('    ' + line)
 435                         _log.warning('')
 436             return False
 437
 438         return True
 439
 440     def _wdiff_missing_message(self):
 441         return 'wdiff is not installed; please install it to generate word-by-word diffs.'
 442
 443     def check_httpd(self):
 444         httpd_path = self.path_to_apache()
 445         try:
 446             server_name = self._filesystem.basename(httpd_path)
 447             env = self.setup_environ_for_server(server_name)
 448             if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0:
 449                 _log.error("httpd seems broken. Cannot run http tests.")
 450                 return False
 451             return True
 452         except OSError:
 453             _log.error("No httpd found. Cannot run http tests.")
 454             return False
 455
 456     def do_text_results_differ(self, expected_text, actual_text):
 457         return expected_text != actual_text
 458
 459     def do_audio_results_differ(self, expected_audio, actual_audio):
 460         return expected_audio != actual_audio
 461
 462     def diff_image(self, expected_contents, actual_contents):
 463         """Compare two images and return a tuple of an image diff, and an error string.
 464
 465         If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff).
 466         """
 467         # If only one of them exists, return that one.
 468         if not actual_contents and not expected_contents:
 469             return (None, None)
 470         if not actual_contents:
 471             return (expected_contents, None)
 472         if not expected_contents:
 473             return (actual_contents, None)
 474
 475         tempdir = self._filesystem.mkdtemp()
 476
 477         expected_filename = self._filesystem.join(str(tempdir), "expected.png")
 478         self._filesystem.write_binary_file(expected_filename, expected_contents)
 479
 480         actual_filename = self._filesystem.join(str(tempdir), "actual.png")
 481         self._filesystem.write_binary_file(actual_filename, actual_contents)
 482
 483         diff_filename = self._filesystem.join(str(tempdir), "diff.png")
 484
 485         # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin.
 486         native_expected_filename = self._convert_path(expected_filename)
 487         native_actual_filename = self._convert_path(actual_filename)
 488         native_diff_filename = self._convert_path(diff_filename)
 489
 490         executable = self._path_to_image_diff()
 491         # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'.
 492         comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename]
 493
 494         result = None
 495         err_str = None
 496         try:
 497             exit_code = self._executive.run_command(comand, return_exit_code=True)
 498             if exit_code == 0:
 499                 # The images are the same.
 500                 result = None
 501             elif exit_code == 1:
 502                 result = self._filesystem.read_binary_file(native_diff_filename)
 503             else:
 504                 err_str = "Image diff returned an exit code of %s. See http://crbug.com/278596" % exit_code
 505         except OSError, e:
 506             err_str = 'error running image diff: %s' % str(e)
 507         finally:
 508             self._filesystem.rmtree(str(tempdir))
 509
 510         return (result, err_str or None)
 511
 512     def diff_text(self, expected_text, actual_text, expected_filename, actual_filename):
 513         """Returns a string containing the diff of the two text strings
 514         in 'unified diff' format."""
 515
 516         # The filenames show up in the diff output, make sure they're
 517         # raw bytes and not unicode, so that they don't trigger join()
 518         # trying to decode the input.
 519         def to_raw_bytes(string_value):
 520             if isinstance(string_value, unicode):
 521                 return string_value.encode('utf-8')
 522             return string_value
 523         expected_filename = to_raw_bytes(expected_filename)
 524         actual_filename = to_raw_bytes(actual_filename)
 525         diff = difflib.unified_diff(expected_text.splitlines(True),
 526                                     actual_text.splitlines(True),
 527                                     expected_filename,
 528                                     actual_filename)
 529         return ''.join(diff)
 530
 531     def driver_name(self):
 532         if self.get_option('driver_name'):
 533             return self.get_option('driver_name')
 534         return self.CONTENT_SHELL_NAME
 535
 536     def expected_baselines_by_extension(self, test_name):
 537         """Returns a dict mapping baseline suffix to relative path for each baseline in
 538         a test. For reftests, it returns ".==" or ".!=" instead of the suffix."""
 539         # FIXME: The name similarity between this and expected_baselines() below, is unfortunate.
 540         # We should probably rename them both.
 541         baseline_dict = {}
 542         reference_files = self.reference_files(test_name)
 543         if reference_files:
 544             # FIXME: How should this handle more than one type of reftest?
 545             baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1])
 546
 547         for extension in self.baseline_extensions():
 548             path = self.expected_filename(test_name, extension, return_default=False)
 549             baseline_dict[extension] = self.relative_test_filename(path) if path else path
 550
 551         return baseline_dict
 552
 553     def baseline_extensions(self):
 554         """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'."""
 555         return ('.wav', '.txt', '.png')
 556
 557     def expected_baselines(self, test_name, suffix, all_baselines=False):
 558         """Given a test name, finds where the baseline results are located.
 559
 560         Args:
 561         test_name: name of test file (usually a relative path under LayoutTests/)
 562         suffix: file suffix of the expected results, including dot; e.g.
 563             '.txt' or '.png'.  This should not be None, but may be an empty
 564             string.
 565         all_baselines: If True, return an ordered list of all baseline paths
 566             for the given platform. If False, return only the first one.
 567         Returns
 568         a list of ( platform_dir, results_filename ), where
 569             platform_dir - abs path to the top of the results tree (or test
 570                 tree)
 571             results_filename - relative path from top of tree to the results
 572                 file
 573             (port.join() of the two gives you the full path to the file,
 574                 unless None was returned.)
 575         Return values will be in the format appropriate for the current
 576         platform (e.g., "\\" for path separators on Windows). If the results
 577         file is not found, then None will be returned for the directory,
 578         but the expected relative pathname will still be returned.
 579
 580         This routine is generic but lives here since it is used in
 581         conjunction with the other baseline and filename routines that are
 582         platform specific.
 583         """
 584         baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix
 585         baseline_search_path = self.baseline_search_path()
 586
 587         baselines = []
 588         for platform_dir in baseline_search_path:
 589             if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
 590                 baselines.append((platform_dir, baseline_filename))
 591
 592             if not all_baselines and baselines:
 593                 return baselines
 594
 595         # If it wasn't found in a platform directory, return the expected
 596         # result in the test directory, even if no such file actually exists.
 597         platform_dir = self.layout_tests_dir()
 598         if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
 599             baselines.append((platform_dir, baseline_filename))
 600
 601         if baselines:
 602             return baselines
 603
 604         return [(None, baseline_filename)]
 605
 606     def expected_filename(self, test_name, suffix, return_default=True):
 607         """Given a test name, returns an absolute path to its expected results.
 608
 609         If no expected results are found in any of the searched directories,
 610         the directory in which the test itself is located will be returned.
 611         The return value is in the format appropriate for the platform
 612         (e.g., "\\" for path separators on windows).
 613
 614         Args:
 615         test_name: name of test file (usually a relative path under LayoutTests/)
 616         suffix: file suffix of the expected results, including dot; e.g. '.txt'
 617             or '.png'.  This should not be None, but may be an empty string.
 618         platform: the most-specific directory name to use to build the
 619             search list of directories, e.g., 'win', or
 620             'chromium-cg-mac-leopard' (we follow the WebKit format)
 621         return_default: if True, returns the path to the generic expectation if nothing
 622             else is found; if False, returns None.
 623
 624         This routine is generic but is implemented here to live alongside
 625         the other baseline and filename manipulation routines.
 626         """
 627         # FIXME: The [0] here is very mysterious, as is the destructured return.
 628         platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0]
 629         if platform_dir:
 630             return self._filesystem.join(platform_dir, baseline_filename)
 631
 632         actual_test_name = self.lookup_virtual_test_base(test_name)
 633         if actual_test_name:
 634             return self.expected_filename(actual_test_name, suffix)
 635
 636         if return_default:
 637             return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
 638         return None
 639
 640     def expected_checksum(self, test_name):
 641         """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
 642         png_path = self.expected_filename(test_name, '.png')
 643
 644         if self._filesystem.exists(png_path):
 645             with self._filesystem.open_binary_file_for_reading(png_path) as filehandle:
 646                 return read_checksum_from_png.read_checksum(filehandle)
 647
 648         return None
 649
 650     def expected_image(self, test_name):
 651         """Returns the image we expect the test to produce."""
 652         baseline_path = self.expected_filename(test_name, '.png')
 653         if not self._filesystem.exists(baseline_path):
 654             return None
 655         return self._filesystem.read_binary_file(baseline_path)
 656
 657     def expected_audio(self, test_name):
 658         baseline_path = self.expected_filename(test_name, '.wav')
 659         if not self._filesystem.exists(baseline_path):
 660             return None
 661         return self._filesystem.read_binary_file(baseline_path)
 662
 663     def expected_text(self, test_name):
 664         """Returns the text output we expect the test to produce, or None
 665         if we don't expect there to be any text output.
 666         End-of-line characters are normalized to '\n'."""
 667         # FIXME: DRT output is actually utf-8, but since we don't decode the
 668         # output from DRT (instead treating it as a binary string), we read the
 669         # baselines as a binary string, too.
 670         baseline_path = self.expected_filename(test_name, '.txt')
 671         if not self._filesystem.exists(baseline_path):
 672             return None
 673         text = self._filesystem.read_binary_file(baseline_path)
 674         return text.replace("\r\n", "\n")
 675
 676     def _get_reftest_list(self, test_name):
 677         dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
 678         if dirname not in self._reftest_list:
 679             self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
 680         return self._reftest_list[dirname]
 681
 682     @staticmethod
 683     def _parse_reftest_list(filesystem, test_dirpath):
 684         reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
 685         if not filesystem.isfile(reftest_list_path):
 686             return None
 687         reftest_list_file = filesystem.read_text_file(reftest_list_path)
 688
 689         parsed_list = {}
 690         for line in reftest_list_file.split('\n'):
 691             line = re.sub('#.+$', '', line)
 692             split_line = line.split()
 693             if len(split_line) == 4:
 694                 # FIXME: Probably one of mozilla's extensions in the reftest.list format. Do we need to support this?
 695                 _log.warning("unsupported reftest.list line '%s' in %s" % (line, reftest_list_path))
 696                 continue
 697             if len(split_line) < 3:
 698                 continue
 699             expectation_type, test_file, ref_file = split_line
 700             parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
 701         return parsed_list
 702
 703     def reference_files(self, test_name):
 704         """Return a list of expectation (== or !=) and filename pairs"""
 705
 706         reftest_list = self._get_reftest_list(test_name)
 707         if not reftest_list:
 708             reftest_list = []
 709             for expectation, prefix in (('==', ''), ('!=', '-mismatch')):
 710                 for extention in Port._supported_file_extensions:
 711                     path = self.expected_filename(test_name, prefix + extention)
 712                     if self._filesystem.exists(path):
 713                         reftest_list.append((expectation, path))
 714             return reftest_list
 715
 716         return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), [])  # pylint: disable=E1103
 717
 718     def tests(self, paths):
 719         """Return the list of tests found matching paths."""
 720         tests = self._real_tests(paths)
 721         tests.extend(self._virtual_tests(paths, self.populated_virtual_test_suites()))
 722         return tests
 723
 724     def _real_tests(self, paths):
 725         # When collecting test cases, skip these directories
 726         skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'script-tests', 'reference', 'reftest'])
 727         files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key)
 728         return [self.relative_test_filename(f) for f in files]
 729
 730     # When collecting test cases, we include any file with these extensions.
 731     _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl',
 732                                       '.htm', '.php', '.svg', '.mht', '.pdf'])
 733
 734     @staticmethod
 735     # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
 736     def is_reference_html_file(filesystem, dirname, filename):
 737         if filename.startswith('ref-') or filename.startswith('notref-'):
 738             return True
 739         filename_wihout_ext, unused = filesystem.splitext(filename)
 740         for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
 741             if filename_wihout_ext.endswith(suffix):
 742                 return True
 743         return False
 744
 745     @staticmethod
 746     def _has_supported_extension(filesystem, filename):
 747         """Return true if filename is one of the file extensions we want to run a test on."""
 748         extension = filesystem.splitext(filename)[1]
 749         return extension in Port._supported_file_extensions
 750
 751     @staticmethod
 752     def is_test_file(filesystem, dirname, filename):
 753         return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
 754
 755     ALL_TEST_TYPES = ['audio', 'harness', 'pixel', 'ref', 'text', 'unknown']
 756
 757     def test_type(self, test_name):
 758         fs = self._filesystem
 759         if fs.exists(self.expected_filename(test_name, '.png')):
 760             return 'pixel'
 761         if fs.exists(self.expected_filename(test_name, '.wav')):
 762             return 'audio'
 763         if self.reference_files(test_name):
 764             return 'ref'
 765         txt = self.expected_text(test_name)
 766         if txt:
 767             if 'layer at (0,0) size 800x600' in txt:
 768                 return 'pixel'
 769             for line in txt.splitlines():
 770                 if line.startswith('FAIL') or line.startswith('TIMEOUT') or line.startswith('PASS'):
 771                     return 'harness'
 772             return 'text'
 773         return 'unknown'
 774
 775     def test_key(self, test_name):
 776         """Turns a test name into a list with two sublists, the natural key of the
 777         dirname, and the natural key of the basename.
 778
 779         This can be used when sorting paths so that files in a directory.
 780         directory are kept together rather than being mixed in with files in
 781         subdirectories."""
 782         dirname, basename = self.split_test(test_name)
 783         return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename))
 784
 785     def _natural_sort_key(self, string_to_split):
 786         """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
 787
 788         This can be used to implement "natural sort" order. See:
 789         http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
 790         http://nedbatchelder.com/blog/200712.html#e20071211T054956
 791         """
 792         def tryint(val):
 793             try:
 794                 return int(val)
 795             except ValueError:
 796                 return val
 797
 798         return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
 799
 800     def test_dirs(self):
 801         """Returns the list of top-level test directories."""
 802         layout_tests_dir = self.layout_tests_dir()
 803         return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
 804                       self._filesystem.listdir(layout_tests_dir))
 805
 806     @memoized
 807     def test_isfile(self, test_name):
 808         """Return True if the test name refers to a directory of tests."""
 809         # Used by test_expectations.py to apply rules to whole directories.
 810         if self._filesystem.isfile(self.abspath_for_test(test_name)):
 811             return True
 812         base = self.lookup_virtual_test_base(test_name)
 813         return base and self._filesystem.isfile(self.abspath_for_test(base))
 814
 815     @memoized
 816     def test_isdir(self, test_name):
 817         """Return True if the test name refers to a directory of tests."""
 818         # Used by test_expectations.py to apply rules to whole directories.
 819         if self._filesystem.isdir(self.abspath_for_test(test_name)):
 820             return True
 821         base = self.lookup_virtual_test_base(test_name)
 822         return base and self._filesystem.isdir(self.abspath_for_test(base))
 823
 824     @memoized
 825     def test_exists(self, test_name):
 826         """Return True if the test name refers to an existing test or baseline."""
 827         # Used by test_expectations.py to determine if an entry refers to a
 828         # valid test and by printing.py to determine if baselines exist.
 829         return self.test_isfile(test_name) or self.test_isdir(test_name)
 830
 831     def split_test(self, test_name):
 832         """Splits a test name into the 'directory' part and the 'basename' part."""
 833         index = test_name.rfind(self.TEST_PATH_SEPARATOR)
 834         if index < 1:
 835             return ('', test_name)
 836         return (test_name[0:index], test_name[index:])
 837
 838     def normalize_test_name(self, test_name):
 839         """Returns a normalized version of the test name or test directory."""
 840         if test_name.endswith('/'):
 841             return test_name
 842         if self.test_isdir(test_name):
 843             return test_name + '/'
 844         return test_name
 845
 846     def driver_cmd_line(self):
 847         """Prints the DRT command line that will be used."""
 848         driver = self.create_driver(0)
 849         return driver.cmd_line(self.get_option('pixel_tests'), [])
 850
 851     def update_baseline(self, baseline_path, data):
 852         """Updates the baseline for a test.
 853
 854         Args:
 855             baseline_path: the actual path to use for baseline, not the path to
 856               the test. This function is used to update either generic or
 857               platform-specific baselines, but we can't infer which here.
 858             data: contents of the baseline.
 859         """
 860         self._filesystem.write_binary_file(baseline_path, data)
 861
 862     # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected).
 863     def webkit_base(self):
 864         return self._webkit_finder.webkit_base()
 865
 866     def path_from_webkit_base(self, *comps):
 867         return self._webkit_finder.path_from_webkit_base(*comps)
 868
 869     def path_from_chromium_base(self, *comps):
 870         return self._webkit_finder.path_from_chromium_base(*comps)
 871
 872     def path_to_script(self, script_name):
 873         return self._webkit_finder.path_to_script(script_name)
 874
 875     def layout_tests_dir(self):
 876         return self._webkit_finder.layout_tests_dir()
 877
 878     def perf_tests_dir(self):
 879         return self._webkit_finder.perf_tests_dir()
 880
 881     def skipped_layout_tests(self, test_list):
 882         """Returns tests skipped outside of the TestExpectations files."""
 883         return set(self._skipped_tests_for_unsupported_features(test_list))
 884
 885     def _tests_from_skipped_file_contents(self, skipped_file_contents):
 886         tests_to_skip = []
 887         for line in skipped_file_contents.split('\n'):
 888             line = line.strip()
 889             line = line.rstrip('/')  # Best to normalize directory names to not include the trailing slash.
 890             if line.startswith('#') or not len(line):
 891                 continue
 892             tests_to_skip.append(line)
 893         return tests_to_skip
 894
 895     def _expectations_from_skipped_files(self, skipped_file_paths):
 896         tests_to_skip = []
 897         for search_path in skipped_file_paths:
 898             filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped")
 899             if not self._filesystem.exists(filename):
 900                 _log.debug("Skipped does not exist: %s" % filename)
 901                 continue
 902             _log.debug("Using Skipped file: %s" % filename)
 903             skipped_file_contents = self._filesystem.read_text_file(filename)
 904             tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
 905         return tests_to_skip
 906
 907     @memoized
 908     def skipped_perf_tests(self):
 909         return self._expectations_from_skipped_files([self.perf_tests_dir()])
 910
 911     def skips_perf_test(self, test_name):
 912         for test_or_category in self.skipped_perf_tests():
 913             if test_or_category == test_name:
 914                 return True
 915             category = self._filesystem.join(self.perf_tests_dir(), test_or_category)
 916             if self._filesystem.isdir(category) and test_name.startswith(test_or_category):
 917                 return True
 918         return False
 919
 920     def is_chromium(self):
 921         return True
 922
 923     def name(self):
 924         """Returns a name that uniquely identifies this particular type of port
 925         (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed
 926         to factory.get() to instantiate the port."""
 927         return self._name
 928
 929     def operating_system(self):
 930         # Subclasses should override this default implementation.
 931         return 'mac'
 932
 933     def version(self):
 934         """Returns a string indicating the version of a given platform, e.g.
 935         'leopard' or 'xp'.
 936
 937         This is used to help identify the exact port when parsing test
 938         expectations, determining search paths, and logging information."""
 939         return self._version
 940
 941     def architecture(self):
 942         return self._architecture
 943
 944     def get_option(self, name, default_value=None):
 945         return getattr(self._options, name, default_value)
 946
 947     def set_option_default(self, name, default_value):
 948         return self._options.ensure_value(name, default_value)
 949
 950     @memoized
 951     def path_to_generic_test_expectations_file(self):
 952         return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations')
 953
 954     def relative_test_filename(self, filename):
 955         """Returns a test_name a relative unix-style path for a filename under the LayoutTests
 956         directory. Ports may legitimately return abspaths here if no relpath makes sense."""
 957         # Ports that run on windows need to override this method to deal with
 958         # filenames with backslashes in them.
 959         if filename.startswith(self.layout_tests_dir()):
 960             return self.host.filesystem.relpath(filename, self.layout_tests_dir())
 961         else:
 962             return self.host.filesystem.abspath(filename)
 963
 964     @memoized
 965     def abspath_for_test(self, test_name):
 966         """Returns the full path to the file for a given test name. This is the
 967         inverse of relative_test_filename()."""
 968         return self._filesystem.join(self.layout_tests_dir(), test_name)
 969
 970     def results_directory(self):
 971         """Absolute path to the place to store the test results (uses --results-directory)."""
 972         if not self._results_directory:
 973             option_val = self.get_option('results_directory') or self.default_results_directory()
 974             self._results_directory = self._filesystem.abspath(option_val)
 975         return self._results_directory
 976
 977     def perf_results_directory(self):
 978         return self._build_path()
 979
 980     def default_results_directory(self):
 981         """Absolute path to the default place to store the test results."""
 982         try:
 983             return self.path_from_chromium_base('webkit', self.get_option('configuration'), 'layout-test-results')
 984         except AssertionError:
 985             return self._build_path('layout-test-results')
 986
 987     def setup_test_run(self):
 988         """Perform port-specific work at the beginning of a test run."""
 989         # Delete the disk cache if any to ensure a clean test run.
 990         dump_render_tree_binary_path = self._path_to_driver()
 991         cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
 992         cachedir = self._filesystem.join(cachedir, "cache")
 993         if self._filesystem.exists(cachedir):
 994             self._filesystem.rmtree(cachedir)
 995
 996         if self._dump_reader:
 997             self._filesystem.maybe_make_directory(self._dump_reader.crash_dumps_directory())
 998
 999     def num_workers(self, requested_num_workers):
1000         """Returns the number of available workers (possibly less than the number requested)."""
1001         return requested_num_workers
1002
1003     def clean_up_test_run(self):
1004         """Perform port-specific work at the end of a test run."""
1005         if self._image_differ:
1006             self._image_differ.stop()
1007             self._image_differ = None
1008
1009     # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable.
1010     def _value_or_default_from_environ(self, name, default=None):
1011         if name in os.environ:
1012             return os.environ[name]
1013         return default
1014
1015     def _copy_value_from_environ_if_set(self, clean_env, name):
1016         if name in os.environ:
1017             clean_env[name] = os.environ[name]
1018
1019     def setup_environ_for_server(self, server_name=None):
1020         # We intentionally copy only a subset of os.environ when
1021         # launching subprocesses to ensure consistent test results.
1022         clean_env = {
1023             'LOCAL_RESOURCE_ROOT': self.layout_tests_dir(),  # FIXME: Is this used?
1024         }
1025         variables_to_copy = [
1026             'WEBKIT_TESTFONTS',  # FIXME: Is this still used?
1027             'WEBKITOUTPUTDIR',   # FIXME: Is this still used?
1028             'CHROME_DEVEL_SANDBOX',
1029             'CHROME_IPC_LOGGING',
1030             'ASAN_OPTIONS',
1031             'VALGRIND_LIB',
1032             'VALGRIND_LIB_INNER',
1033         ]
1034         if self.host.platform.is_linux() or self.host.platform.is_freebsd():
1035             variables_to_copy += [
1036                 'XAUTHORITY',
1037                 'HOME',
1038                 'LANG',
1039                 'LD_LIBRARY_PATH',
1040                 'DBUS_SESSION_BUS_ADDRESS',
1041                 'XDG_DATA_DIRS',
1042             ]
1043             clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1')
1044         if self.host.platform.is_mac():
1045             clean_env['DYLD_LIBRARY_PATH'] = self._build_path()
1046             clean_env['DYLD_FRAMEWORK_PATH'] = self._build_path()
1047             variables_to_copy += [
1048                 'HOME',
1049             ]
1050         if self.host.platform.is_win():
1051             variables_to_copy += [
1052                 'PATH',
1053                 'GYP_DEFINES',  # Required to locate win sdk.
1054             ]
1055         if self.host.platform.is_cygwin():
1056             variables_to_copy += [
1057                 'HOMEDRIVE',
1058                 'HOMEPATH',
1059                 '_NT_SYMBOL_PATH',
1060             ]
1061
1062         for variable in variables_to_copy:
1063             self._copy_value_from_environ_if_set(clean_env, variable)
1064
1065         for string_variable in self.get_option('additional_env_var', []):
1066             [name, value] = string_variable.split('=', 1)
1067             clean_env[name] = value
1068
1069         return clean_env
1070
1071     def show_results_html_file(self, results_filename):
1072         """This routine should display the HTML file pointed at by
1073         results_filename in a users' browser."""
1074         return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename))
1075
1076     def create_driver(self, worker_number, no_timeout=False):
1077         """Return a newly created Driver subclass for starting/stopping the test driver."""
1078         return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout)
1079
1080     def start_helper(self):
1081         """If a port needs to reconfigure graphics settings or do other
1082         things to ensure a known test configuration, it should override this
1083         method."""
1084         helper_path = self._path_to_helper()
1085         if helper_path:
1086             _log.debug("Starting layout helper %s" % helper_path)
1087             # Note: Not thread safe: http://bugs.python.org/issue2320
1088             self._helper = self._executive.popen([helper_path],
1089                 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
1090             is_ready = self._helper.stdout.readline()
1091             if not is_ready.startswith('ready'):
1092                 _log.error("layout_test_helper failed to be ready")
1093
1094     def requires_http_server(self):
1095         """Does the port require an HTTP server for running tests? This could
1096         be the case when the tests aren't run on the host platform."""
1097         return False
1098
1099     def start_http_server(self, additional_dirs, number_of_drivers):
1100         """Start a web server. Raise an error if it can't start or is already running.
1101
1102         Ports can stub this out if they don't need a web server to be running."""
1103         assert not self._http_server, 'Already running an http server.'
1104
1105         server = apache_http.ApacheHTTP(self, self.results_directory(),
1106                                         additional_dirs=additional_dirs,
1107                                         number_of_servers=(number_of_drivers * 4))
1108         server.start()
1109         self._http_server = server
1110
1111     def start_websocket_server(self):
1112         """Start a web server. Raise an error if it can't start or is already running.
1113
1114         Ports can stub this out if they don't need a websocket server to be running."""
1115         assert not self._websocket_server, 'Already running a websocket server.'
1116
1117         server = pywebsocket.PyWebSocket(self, self.results_directory())
1118         server.start()
1119         self._websocket_server = server
1120
1121     def http_server_supports_ipv6(self):
1122         # Apache < 2.4 on win32 does not support IPv6, nor does cygwin apache.
1123         if self.host.platform.is_cygwin() or self.host.platform.is_win():
1124             return False
1125         return True
1126
1127     def stop_helper(self):
1128         """Shut down the test helper if it is running. Do nothing if
1129         it isn't, or it isn't available. If a port overrides start_helper()
1130         it must override this routine as well."""
1131         if self._helper:
1132             _log.debug("Stopping layout test helper")
1133             try:
1134                 self._helper.stdin.write("x\n")
1135                 self._helper.stdin.close()
1136                 self._helper.wait()
1137             except IOError, e:
1138                 pass
1139             finally:
1140                 self._helper = None
1141
1142     def stop_http_server(self):
1143         """Shut down the http server if it is running. Do nothing if it isn't."""
1144         if self._http_server:
1145             self._http_server.stop()
1146             self._http_server = None
1147
1148     def stop_websocket_server(self):
1149         """Shut down the websocket server if it is running. Do nothing if it isn't."""
1150         if self._websocket_server:
1151             self._websocket_server.stop()
1152             self._websocket_server = None
1153
1154     #
1155     # TEST EXPECTATION-RELATED METHODS
1156     #
1157
1158     def test_configuration(self):
1159         """Returns the current TestConfiguration for the port."""
1160         if not self._test_configuration:
1161             self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower())
1162         return self._test_configuration
1163
1164     # FIXME: Belongs on a Platform object.
1165     @memoized
1166     def all_test_configurations(self):
1167         """Returns a list of TestConfiguration instances, representing all available
1168         test configurations for this port."""
1169         return self._generate_all_test_configurations()
1170
1171     # FIXME: Belongs on a Platform object.
1172     def configuration_specifier_macros(self):
1173         """Ports may provide a way to abbreviate configuration specifiers to conveniently
1174         refer to them as one term or alias specific values to more generic ones. For example:
1175
1176         (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake.
1177         (lucid) -> linux  # Change specific name of the Linux distro to a more generic term.
1178
1179         Returns a dictionary, each key representing a macro term ('win', for example),
1180         and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7'])."""
1181         return self.CONFIGURATION_SPECIFIER_MACROS
1182
1183     def all_baseline_variants(self):
1184         """Returns a list of platform names sufficient to cover all the baselines.
1185
1186         The list should be sorted so that a later platform  will reuse
1187         an earlier platform's baselines if they are the same (e.g.,
1188         'snowleopard' should precede 'leopard')."""
1189         return self.ALL_BASELINE_VARIANTS
1190
1191     def _generate_all_test_configurations(self):
1192         """Returns a sequence of the TestConfigurations the port supports."""
1193         # By default, we assume we want to test every graphics type in
1194         # every configuration on every system.
1195         test_configurations = []
1196         for version, architecture in self.ALL_SYSTEMS:
1197             for build_type in self.ALL_BUILD_TYPES:
1198                 test_configurations.append(TestConfiguration(version, architecture, build_type))
1199         return test_configurations
1200
1201     try_builder_names = frozenset([
1202         'linux_layout',
1203         'mac_layout',
1204         'win_layout',
1205         'linux_layout_rel',
1206         'mac_layout_rel',
1207         'win_layout_rel',
1208     ])
1209
1210     def warn_if_bug_missing_in_test_expectations(self):
1211         return True
1212
1213     def _port_specific_expectations_files(self):
1214         paths = []
1215         paths.append(self.path_from_chromium_base('skia', 'skia_test_expectations.txt'))
1216         paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations_w3c.txt'))
1217         paths.append(self._filesystem.join(self.layout_tests_dir(), 'NeverFixTests'))
1218         paths.append(self._filesystem.join(self.layout_tests_dir(), 'StaleTestExpectations'))
1219         paths.append(self._filesystem.join(self.layout_tests_dir(), 'SlowTests'))
1220         paths.append(self._filesystem.join(self.layout_tests_dir(), 'FlakyTests'))
1221
1222         builder_name = self.get_option('builder_name', 'DUMMY_BUILDER_NAME')
1223         if builder_name == 'DUMMY_BUILDER_NAME' or '(deps)' in builder_name or builder_name in self.try_builder_names:
1224             paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations.txt'))
1225         return paths
1226
1227     def expectations_dict(self):
1228         """Returns an OrderedDict of name -> expectations strings.
1229         The names are expected to be (but not required to be) paths in the filesystem.
1230         If the name is a path, the file can be considered updatable for things like rebaselining,
1231         so don't use names that are paths if they're not paths.
1232         Generally speaking the ordering should be files in the filesystem in cascade order
1233         (TestExpectations followed by Skipped, if the port honors both formats),
1234         then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options."""
1235         # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict.
1236         expectations = OrderedDict()
1237
1238         for path in self.expectations_files():
1239             if self._filesystem.exists(path):
1240                 expectations[path] = self._filesystem.read_text_file(path)
1241
1242         for path in self.get_option('additional_expectations', []):
1243             expanded_path = self._filesystem.expanduser(path)
1244             if self._filesystem.exists(expanded_path):
1245                 _log.debug("reading additional_expectations from path '%s'" % path)
1246                 expectations[path] = self._filesystem.read_text_file(expanded_path)
1247             else:
1248                 _log.warning("additional_expectations path '%s' does not exist" % path)
1249         return expectations
1250
1251     def bot_expectations(self):
1252         if not self.get_option('ignore_flaky_tests'):
1253             return {}
1254
1255         full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name)
1256         builder_category = self.get_option('ignore_builder_category', 'layout')
1257         factory = BotTestExpectationsFactory()
1258         # FIXME: This only grabs release builder's flakiness data. If we're running debug,
1259         # when we should grab the debug builder's data.
1260         expectations = factory.expectations_for_port(full_port_name, builder_category)
1261
1262         if not expectations:
1263             return {}
1264
1265         ignore_mode = self.get_option('ignore_flaky_tests')
1266         if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky':
1267             return expectations.flakes_by_path(ignore_mode == 'very-flaky')
1268         if ignore_mode == 'unexpected':
1269             return expectations.unexpected_results_by_path()
1270         _log.warning("Unexpected ignore mode: '%s'." % ignore_mode)
1271         return {}
1272
1273     def expectations_files(self):
1274         return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files()
1275
1276     def repository_paths(self):
1277         """Returns a list of (repository_name, repository_path) tuples of its depending code base."""
1278         return [('blink', self.layout_tests_dir()),
1279                 ('chromium', self.path_from_chromium_base('build'))]
1280
1281     _WDIFF_DEL = '##WDIFF_DEL##'
1282     _WDIFF_ADD = '##WDIFF_ADD##'
1283     _WDIFF_END = '##WDIFF_END##'
1284
1285     def _format_wdiff_output_as_html(self, wdiff):
1286         wdiff = cgi.escape(wdiff)
1287         wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
1288         wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
1289         wdiff = wdiff.replace(self._WDIFF_END, "</span>")
1290         html = "<head><style>.del { background: #faa; } "
1291         html += ".add { background: #afa; }</style></head>"
1292         html += "<pre>%s</pre>" % wdiff
1293         return html
1294
1295     def _wdiff_command(self, actual_filename, expected_filename):
1296         executable = self._path_to_wdiff()
1297         return [executable,
1298                 "--start-delete=%s" % self._WDIFF_DEL,
1299                 "--end-delete=%s" % self._WDIFF_END,
1300                 "--start-insert=%s" % self._WDIFF_ADD,
1301                 "--end-insert=%s" % self._WDIFF_END,
1302                 actual_filename,
1303                 expected_filename]
1304
1305     @staticmethod
1306     def _handle_wdiff_error(script_error):
1307         # Exit 1 means the files differed, any other exit code is an error.
1308         if script_error.exit_code != 1:
1309             raise script_error
1310
1311     def _run_wdiff(self, actual_filename, expected_filename):
1312         """Runs wdiff and may throw exceptions.
1313         This is mostly a hook for unit testing."""
1314         # Diffs are treated as binary as they may include multiple files
1315         # with conflicting encodings.  Thus we do not decode the output.
1316         command = self._wdiff_command(actual_filename, expected_filename)
1317         wdiff = self._executive.run_command(command, decode_output=False,
1318             error_handler=self._handle_wdiff_error)
1319         return self._format_wdiff_output_as_html(wdiff)
1320
1321     _wdiff_error_html = "Failed to run wdiff, see error log."
1322
1323     def wdiff_text(self, actual_filename, expected_filename):
1324         """Returns a string of HTML indicating the word-level diff of the
1325         contents of the two filenames. Returns an empty string if word-level
1326         diffing isn't available."""
1327         if not self.wdiff_available():
1328             return ""
1329         try:
1330             # It's possible to raise a ScriptError we pass wdiff invalid paths.
1331             return self._run_wdiff(actual_filename, expected_filename)
1332         except OSError as e:
1333             if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
1334                 # Silently ignore cases where wdiff is missing.
1335                 self._wdiff_available = False
1336                 return ""
1337             raise
1338         except ScriptError as e:
1339             _log.error("Failed to run wdiff: %s" % e)
1340             self._wdiff_available = False
1341             return self._wdiff_error_html
1342
1343     # This is a class variable so we can test error output easily.
1344     _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
1345
1346     def pretty_patch_text(self, diff_path):
1347         if self._pretty_patch_available is None:
1348             self._pretty_patch_available = self.check_pretty_patch(logging=False)
1349         if not self._pretty_patch_available:
1350             return self._pretty_patch_error_html
1351         command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path),
1352                    self._pretty_patch_path, diff_path)
1353         try:
1354             # Diffs are treated as binary (we pass decode_output=False) as they
1355             # may contain multiple files of conflicting encodings.
1356             return self._executive.run_command(command, decode_output=False)
1357         except OSError, e:
1358             # If the system is missing ruby log the error and stop trying.
1359             self._pretty_patch_available = False
1360             _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
1361             return self._pretty_patch_error_html
1362         except ScriptError, e:
1363             # If ruby failed to run for some reason, log the command
1364             # output and stop trying.
1365             self._pretty_patch_available = False
1366             _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output()))
1367             return self._pretty_patch_error_html
1368
1369     def default_configuration(self):
1370         return self._config.default_configuration()
1371
1372     def clobber_old_port_specific_results(self):
1373         pass
1374
1375     # FIXME: This does not belong on the port object.
1376     @memoized
1377     def path_to_apache(self):
1378         """Returns the full path to the apache binary.
1379
1380         This is needed only by ports that use the apache_http_server module."""
1381         raise NotImplementedError('Port.path_to_apache')
1382
1383     def path_to_apache_config_file(self):
1384         """Returns the full path to the apache configuration file.
1385
1386         If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its
1387         contents will be used instead.
1388
1389         This is needed only by ports that use the apache_http_server module."""
1390         config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH')
1391         if config_file_from_env:
1392             if not self._filesystem.exists(config_file_from_env):
1393                 raise IOError('%s was not found on the system' % config_file_from_env)
1394             return config_file_from_env
1395
1396         config_file_name = self._apache_config_file_name_for_platform(sys.platform)
1397         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
1398
1399     #
1400     # PROTECTED ROUTINES
1401     #
1402     # The routines below should only be called by routines in this class
1403     # or any of its subclasses.
1404     #
1405
1406     # FIXME: This belongs on some platform abstraction instead of Port.
1407     def _is_redhat_based(self):
1408         return self._filesystem.exists('/etc/redhat-release')
1409
1410     def _is_debian_based(self):
1411         return self._filesystem.exists('/etc/debian_version')
1412
1413     def _apache_version(self):
1414         config = self._executive.run_command([self.path_to_apache(), '-v'])
1415         return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config)
1416
1417     # We pass sys_platform into this method to make it easy to unit test.
1418     def _apache_config_file_name_for_platform(self, sys_platform):
1419         if sys_platform == 'cygwin':
1420             return 'cygwin-httpd.conf'  # CYGWIN is the only platform to still use Apache 1.3.
1421         if sys_platform.startswith('linux'):
1422             if self._is_redhat_based():
1423                 return 'fedora-httpd-' + self._apache_version() + '.conf'
1424             if self._is_debian_based():
1425                 return 'debian-httpd-' + self._apache_version() + '.conf'
1426         # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
1427         return "apache2-httpd.conf"
1428
1429     def _path_to_driver(self, configuration=None):
1430         """Returns the full path to the test driver."""
1431         return self._build_path(self.driver_name())
1432
1433     def _path_to_webcore_library(self):
1434         """Returns the full path to a built copy of WebCore."""
1435         return None
1436
1437     def _path_to_helper(self):
1438         """Returns the full path to the layout_test_helper binary, which
1439         is used to help configure the system for the test run, or None
1440         if no helper is needed.
1441
1442         This is likely only used by start/stop_helper()."""
1443         return None
1444
1445     def _path_to_image_diff(self):
1446         """Returns the full path to the image_diff binary, or None if it is not available.
1447
1448         This is likely used only by diff_image()"""
1449         return self._build_path('image_diff')
1450
1451     @memoized
1452     def _path_to_wdiff(self):
1453         """Returns the full path to the wdiff binary, or None if it is not available.
1454
1455         This is likely used only by wdiff_text()"""
1456         for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"):
1457             if self._filesystem.exists(path):
1458                 return path
1459         return None
1460
1461     def _webkit_baseline_path(self, platform):
1462         """Return the  full path to the top of the baseline tree for a
1463         given platform."""
1464         return self._filesystem.join(self.layout_tests_dir(), 'platform', platform)
1465
1466     def _driver_class(self):
1467         """Returns the port's driver implementation."""
1468         return driver.Driver
1469
1470     def _output_contains_sanitizer_messages(self, output):
1471         if not output:
1472             return None
1473         if 'AddressSanitizer' in output:
1474             return 'AddressSanitizer'
1475         if 'MemorySanitizer' in output:
1476             return 'MemorySanitizer'
1477         return None
1478
1479     def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
1480         if self._output_contains_sanitizer_messages(stderr):
1481             # Running the symbolizer script can take a lot of memory, so we need to
1482             # serialize access to it across all the concurrently running drivers.
1483
1484             # FIXME: investigate using LLVM_SYMBOLIZER_PATH here to reduce the overhead.
1485             sanitizer_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
1486             sanitizer_strip_path_prefix = 'Release/../../'
1487             if self._filesystem.exists(sanitizer_filter_path):
1488                 stderr = self._executive.run_command(['flock', sys.executable, sanitizer_filter_path, sanitizer_strip_path_prefix], input=stderr, decode_output=False)
1489
1490         name_str = name or '<unknown process name>'
1491         pid_str = str(pid or '<unknown>')
1492         stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines()
1493         stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines()
1494         return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str,
1495             '\n'.join(('STDOUT: ' + l) for l in stdout_lines),
1496             '\n'.join(('STDERR: ' + l) for l in stderr_lines)))
1497
1498     def look_for_new_crash_logs(self, crashed_processes, start_time):
1499         pass
1500
1501     def look_for_new_samples(self, unresponsive_processes, start_time):
1502         pass
1503
1504     def sample_process(self, name, pid):
1505         pass
1506
1507     def physical_test_suites(self):
1508         return [
1509             # For example, to turn on force-compositing-mode in the svg/ directory:
1510             # PhysicalTestSuite('svg',
1511             #                   ['--force-compositing-mode']),
1512             ]
1513
1514     def virtual_test_suites(self):
1515         return [
1516             VirtualTestSuite('gpu',
1517                              'fast/canvas',
1518                              ['--enable-accelerated-2d-canvas']),
1519             VirtualTestSuite('gpu',
1520                              'canvas/philip',
1521                              ['--enable-accelerated-2d-canvas']),
1522             VirtualTestSuite('threaded',
1523                              'compositing/visibility',
1524                              ['--enable-threaded-compositing']),
1525             VirtualTestSuite('threaded',
1526                              'compositing/webgl',
1527                              ['--enable-threaded-compositing']),
1528             VirtualTestSuite('softwarecompositing',
1529                              'compositing',
1530                              ['--disable-gpu',
1531                               '--disable-gpu-compositing'],
1532                              use_legacy_naming=True),
1533             VirtualTestSuite('deferred',
1534                              'fast/images',
1535                              ['--enable-deferred-image-decoding',
1536                               '--enable-per-tile-painting']),
1537             VirtualTestSuite('deferred',
1538                              'inspector/timeline',
1539                              ['--enable-deferred-image-decoding',
1540                               '--enable-per-tile-painting']),
1541             VirtualTestSuite('gpu/compositedscrolling/overflow',
1542                              'compositing/overflow',
1543                              ['--enable-accelerated-overflow-scroll'],
1544                              use_legacy_naming=True),
1545             VirtualTestSuite('gpu/compositedscrolling/scrollbars',
1546                              'scrollbars',
1547                              ['--enable-accelerated-overflow-scroll'],
1548                              use_legacy_naming=True),
1549             VirtualTestSuite('threaded',
1550                              'animations',
1551                              ['--enable-threaded-compositing']),
1552             VirtualTestSuite('threaded',
1553                              'transitions',
1554                              ['--enable-threaded-compositing']),
1555             VirtualTestSuite('stable',
1556                              'webexposed',
1557                              ['--stable-release-mode']),
1558             VirtualTestSuite('stable',
1559                              'animations-unprefixed',
1560                              ['--stable-release-mode']),
1561             VirtualTestSuite('stable',
1562                              'media/stable',
1563                              ['--stable-release-mode']),
1564             VirtualTestSuite('android',
1565                              'fullscreen',
1566                              ['--enable-threaded-compositing',
1567                               '--enable-fixed-position-compositing', '--enable-accelerated-overflow-scroll', '--enable-accelerated-scrollable-frames',
1568                               '--enable-composited-scrolling-for-frames', '--enable-gesture-tap-highlight', '--enable-pinch',
1569                               '--enable-overlay-fullscreen-video', '--enable-overlay-scrollbars', '--enable-overscroll-notifications',
1570                               '--enable-fixed-layout', '--enable-viewport', '--disable-canvas-aa',
1571                               '--disable-composited-antialiasing', '--enable-accelerated-fixed-root-background']),
1572             VirtualTestSuite('implsidepainting',
1573                              'inspector/timeline',
1574                              ['--enable-threaded-compositing', '--enable-impl-side-painting']),
1575             VirtualTestSuite('stable',
1576                              'fast/css3-text/css3-text-decoration/stable',
1577                              ['--stable-release-mode']),
1578             VirtualTestSuite('stable',
1579                              'web-animations-api',
1580                              ['--stable-release-mode']),
1581             VirtualTestSuite('linux-subpixel',
1582                              'platform/linux/fast/text/subpixel',
1583                              ['--enable-webkit-text-subpixel-positioning']),
1584             VirtualTestSuite('antialiasedtext',
1585                              'fast/text',
1586                              ['--enable-direct-write',
1587                               '--enable-font-antialiasing']),
1588             VirtualTestSuite('threaded',
1589                              'printing',
1590                              ['--enable-threaded-compositing']),
1591             VirtualTestSuite('regionbasedmulticol',
1592                              'fast/multicol',
1593                              ['--enable-region-based-columns']),
1594             VirtualTestSuite('regionbasedmulticol',
1595                              'fast/pagination',
1596                              ['--enable-region-based-columns']),
1597         ]
1598
1599     @memoized
1600     def populated_virtual_test_suites(self):
1601         suites = self.virtual_test_suites()
1602
1603         # Sanity-check the suites to make sure they don't point to other suites.
1604         suite_dirs = [suite.name for suite in suites]
1605         for suite in suites:
1606             assert suite.base not in suite_dirs
1607
1608         for suite in suites:
1609             base_tests = self._real_tests([suite.base])
1610             suite.tests = {}
1611             for test in base_tests:
1612                 suite.tests[test.replace(suite.base, suite.name, 1)] = test
1613         return suites
1614
1615     def _virtual_tests(self, paths, suites):
1616         virtual_tests = list()
1617         for suite in suites:
1618             if paths:
1619                 for test in suite.tests:
1620                     if any(test.startswith(p) for p in paths):
1621                         virtual_tests.append(test)
1622             else:
1623                 virtual_tests.extend(suite.tests.keys())
1624         return virtual_tests
1625
1626     def is_virtual_test(self, test_name):
1627         return bool(self.lookup_virtual_suite(test_name))
1628
1629     def lookup_virtual_suite(self, test_name):
1630         for suite in self.populated_virtual_test_suites():
1631             if test_name.startswith(suite.name):
1632                 return suite
1633         return None
1634
1635     def lookup_virtual_test_base(self, test_name):
1636         suite = self.lookup_virtual_suite(test_name)
1637         if not suite:
1638             return None
1639         return test_name.replace(suite.name, suite.base, 1)
1640
1641     def lookup_virtual_test_args(self, test_name):
1642         for suite in self.populated_virtual_test_suites():
1643             if test_name.startswith(suite.name):
1644                 return suite.args
1645         return []
1646
1647     def lookup_physical_test_args(self, test_name):
1648         for suite in self.physical_test_suites():
1649             if test_name.startswith(suite.name):
1650                 return suite.args
1651         return []
1652
1653     def should_run_as_pixel_test(self, test_input):
1654         if not self._options.pixel_tests:
1655             return False
1656         if self._options.pixel_test_directories:
1657             return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories)
1658         return True
1659
1660     def _modules_to_search_for_symbols(self):
1661         path = self._path_to_webcore_library()
1662         if path:
1663             return [path]
1664         return []
1665
1666     def _symbols_string(self):
1667         symbols = ''
1668         for path_to_module in self._modules_to_search_for_symbols():
1669             try:
1670                 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error)
1671             except OSError, e:
1672                 _log.warn("Failed to run nm: %s.  Can't determine supported features correctly." % e)
1673         return symbols
1674
1675     # Ports which use compile-time feature detection should define this method and return
1676     # a dictionary mapping from symbol substrings to possibly disabled test directories.
1677     # When the symbol substrings are not matched, the directories will be skipped.
1678     # If ports don't ever enable certain features, then those directories can just be
1679     # in the Skipped list instead of compile-time-checked here.
1680     def _missing_symbol_to_skipped_tests(self):
1681         if self.PORT_HAS_AUDIO_CODECS_BUILT_IN:
1682             return {}
1683         else:
1684             return {
1685                 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"],
1686                 "ff_aac_decoder": ["webaudio/codec-tests/aac"],
1687             }
1688
1689     def _has_test_in_directories(self, directory_lists, test_list):
1690         if not test_list:
1691             return False
1692
1693         directories = itertools.chain.from_iterable(directory_lists)
1694         for directory, test in itertools.product(directories, test_list):
1695             if test.startswith(directory):
1696                 return True
1697         return False
1698
1699     def _skipped_tests_for_unsupported_features(self, test_list):
1700         # Only check the symbols of there are tests in the test_list that might get skipped.
1701         # This is a performance optimization to avoid the calling nm.
1702         # Runtime feature detection not supported, fallback to static detection:
1703         # Disable any tests for symbols missing from the executable or libraries.
1704         if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
1705             symbols_string = self._symbols_string()
1706             if symbols_string is not None:
1707                 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], [])
1708         return []
1709
1710     def _convert_path(self, path):
1711         """Handles filename conversion for subprocess command line args."""
1712         # See note above in diff_image() for why we need this.
1713         if sys.platform == 'cygwin':
1714             return cygpath(path)
1715         return path
1716
1717     def _build_path(self, *comps):
1718         return self._build_path_with_configuration(None, *comps)
1719
1720     def _build_path_with_configuration(self, configuration, *comps):
1721         # Note that we don't do the option caching that the
1722         # base class does, because finding the right directory is relatively
1723         # fast.
1724         configuration = configuration or self.get_option('configuration')
1725         return self._static_build_path(self._filesystem, self.get_option('build_directory'),
1726             self.path_from_chromium_base(), configuration, comps)
1727
1728     def _check_driver_build_up_to_date(self, configuration):
1729         if configuration in ('Debug', 'Release'):
1730             try:
1731                 debug_path = self._path_to_driver('Debug')
1732                 release_path = self._path_to_driver('Release')
1733
1734                 debug_mtime = self._filesystem.mtime(debug_path)
1735                 release_mtime = self._filesystem.mtime(release_path)
1736
1737                 if (debug_mtime > release_mtime and configuration == 'Release' or
1738                     release_mtime > debug_mtime and configuration == 'Debug'):
1739                     most_recent_binary = 'Release' if configuration == 'Debug' else 'Debug'
1740                     _log.warning('You are running the %s binary. However the %s binary appears to be more recent. '
1741                                  'Please pass --%s.', configuration, most_recent_binary, most_recent_binary.lower())
1742                     _log.warning('')
1743             # This will fail if we don't have both a debug and release binary.
1744             # That's fine because, in this case, we must already be running the
1745             # most up-to-date one.
1746             except OSError:
1747                 pass
1748         return True
1749
1750     def _chromium_baseline_path(self, platform):
1751         if platform is None:
1752             platform = self.name()
1753         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
1754
1755 class VirtualTestSuite(object):
1756     def __init__(self, name, base, args, use_legacy_naming=False, tests=None):
1757         if use_legacy_naming:
1758             self.name = 'virtual/' + name
1759         else:
1760             if name.find('/') != -1:
1761                 _log.error("Virtual test suites names cannot contain /'s: %s" % name)
1762                 return
1763             self.name = 'virtual/' + name + '/' + base
1764         self.base = base
1765         self.args = args
1766         self.tests = tests or set()
1767
1768     def __repr__(self):
1769         return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)
1770
1771
1772 class PhysicalTestSuite(object):
1773     def __init__(self, base, args):
1774         self.name = base
1775         self.base = base
1776         self.args = args
1777         self.tests = set()
1778
1779     def __repr__(self):
1780         return "PhysicalTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)