src/third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the Google name nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 """Abstract base class of Port-specific entry points for the layout tests
  30 test infrastructure (the Port and Driver classes)."""
  31
  32 import cgi
  33 import difflib
  34 import errno
  35 import itertools
  36 import logging
  37 import os
  38 import operator
  39 import optparse
  40 import re
  41 import sys
  42
  43 try:
  44     from collections import OrderedDict
  45 except ImportError:
  46     # Needed for Python < 2.7
  47     from webkitpy.thirdparty.ordered_dict import OrderedDict
  48
  49
  50 from webkitpy.common import find_files
  51 from webkitpy.common import read_checksum_from_png
  52 from webkitpy.common.memoized import memoized
  53 from webkitpy.common.system import path
  54 from webkitpy.common.system.executive import ScriptError
  55 from webkitpy.common.system.path import cygpath
  56 from webkitpy.common.system.systemhost import SystemHost
  57 from webkitpy.common.webkit_finder import WebKitFinder
  58 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
  59 from webkitpy.layout_tests.models import test_run_results
  60 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  61 from webkitpy.layout_tests.port import config as port_config
  62 from webkitpy.layout_tests.port import driver
  63 from webkitpy.layout_tests.port import server_process
  64 from webkitpy.layout_tests.port.factory import PortFactory
  65 from webkitpy.layout_tests.servers import apache_http_server
  66 from webkitpy.layout_tests.servers import http_server
  67 from webkitpy.layout_tests.servers import websocket_server
  68
  69 _log = logging.getLogger(__name__)
  70
  71
  72 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports.
  73 class Port(object):
  74     """Abstract class for Port-specific hooks for the layout_test package."""
  75
  76     # Subclasses override this. This should indicate the basic implementation
  77     # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?)
  78     # one unique value per class.
  79
  80     # FIXME: We should probably rename this to something like 'implementation_name'.
  81     port_name = None
  82
  83     # Test names resemble unix relative paths, and use '/' as a directory separator.
  84     TEST_PATH_SEPARATOR = '/'
  85
  86     ALL_BUILD_TYPES = ('debug', 'release')
  87
  88     CONTENT_SHELL_NAME = 'content_shell'
  89
  90     # True if the port as aac and mp3 codecs built in.
  91     PORT_HAS_AUDIO_CODECS_BUILT_IN = False
  92
  93     ALL_SYSTEMS = (
  94         ('snowleopard', 'x86'),
  95         ('lion', 'x86'),
  96
  97         # FIXME: We treat Retina (High-DPI) devices as if they are running
  98         # a different operating system version. This isn't accurate, but will work until
  99         # we need to test and support baselines across multiple O/S versions.
 100         ('retina', 'x86'),
 101
 102         ('mountainlion', 'x86'),
 103         ('mavericks', 'x86'),
 104         ('xp', 'x86'),
 105         ('win7', 'x86'),
 106         ('lucid', 'x86'),
 107         ('lucid', 'x86_64'),
 108         # FIXME: Technically this should be 'arm', but adding a third architecture type breaks TestConfigurationConverter.
 109         # If we need this to be 'arm' in the future, then we first have to fix TestConfigurationConverter.
 110         ('icecreamsandwich', 'x86'),
 111         )
 112
 113     ALL_BASELINE_VARIANTS = [
 114         'mac-mavericks', 'mac-mountainlion', 'mac-retina', 'mac-lion', 'mac-snowleopard',
 115         'win-win7', 'win-xp',
 116         'linux-x86_64', 'linux-x86',
 117     ]
 118
 119     CONFIGURATION_SPECIFIER_MACROS = {
 120         'mac': ['snowleopard', 'lion', 'retina', 'mountainlion', 'mavericks'],
 121         'win': ['xp', 'win7'],
 122         'linux': ['lucid'],
 123         'android': ['icecreamsandwich'],
 124     }
 125
 126     DEFAULT_BUILD_DIRECTORIES = ('out',)
 127
 128     # overridden in subclasses.
 129     FALLBACK_PATHS = {}
 130
 131     SUPPORTED_VERSIONS = []
 132
 133     @classmethod
 134     def latest_platform_fallback_path(cls):
 135         return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]]
 136
 137     @classmethod
 138     def _static_build_path(cls, filesystem, build_directory, chromium_base, configuration, comps):
 139         if build_directory:
 140             return filesystem.join(build_directory, configuration, *comps)
 141
 142         hits = []
 143         for directory in cls.DEFAULT_BUILD_DIRECTORIES:
 144             base_dir = filesystem.join(chromium_base, directory, configuration)
 145             path = filesystem.join(base_dir, *comps)
 146             if filesystem.exists(path):
 147                 hits.append((filesystem.mtime(path), path))
 148
 149         if hits:
 150             hits.sort(reverse=True)
 151             return hits[0][1]  # Return the newest file found.
 152
 153         # We have to default to something, so pick the last one.
 154         return filesystem.join(base_dir, *comps)
 155
 156     @classmethod
 157     def determine_full_port_name(cls, host, options, port_name):
 158         """Return a fully-specified port name that can be used to construct objects."""
 159         # Subclasses will usually override this.
 160         assert port_name.startswith(cls.port_name)
 161         return port_name
 162
 163     def __init__(self, host, port_name, options=None, **kwargs):
 164
 165         # This value may be different from cls.port_name by having version modifiers
 166         # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2').
 167         self._name = port_name
 168
 169         # These are default values that should be overridden in a subclasses.
 170         self._version = ''
 171         self._architecture = 'x86'
 172
 173         # FIXME: Ideally we'd have a package-wide way to get a
 174         # well-formed options object that had all of the necessary
 175         # options defined on it.
 176         self._options = options or optparse.Values()
 177
 178         self.host = host
 179         self._executive = host.executive
 180         self._filesystem = host.filesystem
 181         self._webkit_finder = WebKitFinder(host.filesystem)
 182         self._config = port_config.Config(self._executive, self._filesystem, self.port_name)
 183
 184         self._helper = None
 185         self._http_server = None
 186         self._websocket_server = None
 187         self._image_differ = None
 188         self._server_process_constructor = server_process.ServerProcess  # overridable for testing
 189         self._http_lock = None  # FIXME: Why does this live on the port object?
 190         self._dump_reader = None
 191
 192         # Python's Popen has a bug that causes any pipes opened to a
 193         # process that can't be executed to be leaked.  Since this
 194         # code is specifically designed to tolerate exec failures
 195         # to gracefully handle cases where wdiff is not installed,
 196         # the bug results in a massive file descriptor leak. As a
 197         # workaround, if an exec failure is ever experienced for
 198         # wdiff, assume it's not available.  This will leak one
 199         # file descriptor but that's better than leaking each time
 200         # wdiff would be run.
 201         #
 202         # http://mail.python.org/pipermail/python-list/
 203         #    2008-August/505753.html
 204         # http://bugs.python.org/issue3210
 205         self._wdiff_available = None
 206
 207         # FIXME: prettypatch.py knows this path, why is it copied here?
 208         self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb")
 209         self._pretty_patch_available = None
 210
 211         if not hasattr(options, 'configuration') or not options.configuration:
 212             self.set_option_default('configuration', self.default_configuration())
 213         self._test_configuration = None
 214         self._reftest_list = {}
 215         self._results_directory = None
 216
 217     def buildbot_archives_baselines(self):
 218         return True
 219
 220     def additional_drt_flag(self):
 221         if self.driver_name() == self.CONTENT_SHELL_NAME:
 222             return ['--dump-render-tree']
 223         return []
 224
 225     def supports_per_test_timeout(self):
 226         return False
 227
 228     def default_pixel_tests(self):
 229         return True
 230
 231     def default_smoke_test_only(self):
 232         return False
 233
 234     def default_timeout_ms(self):
 235         timeout_ms = 6 * 1000
 236         if self.get_option('configuration') == 'Debug':
 237             # Debug is usually 2x-3x slower than Release.
 238             return 3 * timeout_ms
 239         return timeout_ms
 240
 241     def driver_stop_timeout(self):
 242         """ Returns the amount of time in seconds to wait before killing the process in driver.stop()."""
 243         # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as
 244         # well (for things like ASAN, Valgrind, etc.)
 245         return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms()
 246
 247     def wdiff_available(self):
 248         if self._wdiff_available is None:
 249             self._wdiff_available = self.check_wdiff(logging=False)
 250         return self._wdiff_available
 251
 252     def pretty_patch_available(self):
 253         if self._pretty_patch_available is None:
 254             self._pretty_patch_available = self.check_pretty_patch(logging=False)
 255         return self._pretty_patch_available
 256
 257     def default_child_processes(self):
 258         """Return the number of drivers to use for this port."""
 259         return self._executive.cpu_count()
 260
 261     def default_max_locked_shards(self):
 262         """Return the number of "locked" shards to run in parallel (like the http tests)."""
 263         max_locked_shards = int(self.default_child_processes()) / 4
 264         if not max_locked_shards:
 265             return 1
 266         return max_locked_shards
 267
 268     def baseline_path(self):
 269         """Return the absolute path to the directory to store new baselines in for this port."""
 270         # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
 271         return self.baseline_version_dir()
 272
 273     def baseline_platform_dir(self):
 274         """Return the absolute path to the default (version-independent) platform-specific results."""
 275         return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
 276
 277     def baseline_version_dir(self):
 278         """Return the absolute path to the platform-and-version-specific results."""
 279         baseline_search_paths = self.baseline_search_path()
 280         return baseline_search_paths[0]
 281
 282     def virtual_baseline_search_path(self, test_name):
 283         suite = self.lookup_virtual_suite(test_name)
 284         if not suite:
 285             return None
 286         return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()]
 287
 288     def baseline_search_path(self):
 289         return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path()
 290
 291     def default_baseline_search_path(self):
 292         """Return a list of absolute paths to directories to search under for
 293         baselines. The directories are searched in order."""
 294         return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version()])
 295
 296     @memoized
 297     def _compare_baseline(self):
 298         factory = PortFactory(self.host)
 299         target_port = self.get_option('compare_port')
 300         if target_port:
 301             return factory.get(target_port).default_baseline_search_path()
 302         return []
 303
 304     def _check_file_exists(self, path_to_file, file_description,
 305                            override_step=None, logging=True):
 306         """Verify the file is present where expected or log an error.
 307
 308         Args:
 309             file_name: The (human friendly) name or description of the file
 310                 you're looking for (e.g., "HTTP Server"). Used for error logging.
 311             override_step: An optional string to be logged if the check fails.
 312             logging: Whether or not log the error messages."""
 313         if not self._filesystem.exists(path_to_file):
 314             if logging:
 315                 _log.error('Unable to find %s' % file_description)
 316                 _log.error('    at %s' % path_to_file)
 317                 if override_step:
 318                     _log.error('    %s' % override_step)
 319                     _log.error('')
 320             return False
 321         return True
 322
 323     def check_build(self, needs_http, printer):
 324         result = True
 325
 326         dump_render_tree_binary_path = self._path_to_driver()
 327         result = self._check_file_exists(dump_render_tree_binary_path,
 328                                          'test driver') and result
 329         if not result and self.get_option('build'):
 330             result = self._check_driver_build_up_to_date(
 331                 self.get_option('configuration'))
 332         else:
 333             _log.error('')
 334
 335         helper_path = self._path_to_helper()
 336         if helper_path:
 337             result = self._check_file_exists(helper_path,
 338                                              'layout test helper') and result
 339
 340         if self.get_option('pixel_tests'):
 341             result = self.check_image_diff(
 342                 'To override, invoke with --no-pixel-tests') and result
 343
 344         # It's okay if pretty patch and wdiff aren't available, but we will at least log messages.
 345         self._pretty_patch_available = self.check_pretty_patch()
 346         self._wdiff_available = self.check_wdiff()
 347
 348         if self._dump_reader:
 349             result = self._dump_reader.check_is_functional() and result
 350
 351         return test_run_results.OK_EXIT_STATUS if result else test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
 352
 353     def _check_driver(self):
 354         driver_path = self._path_to_driver()
 355         if not self._filesystem.exists(driver_path):
 356             _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
 357             return False
 358         return True
 359
 360     def _check_port_build(self):
 361         # Ports can override this method to do additional checks.
 362         return True
 363
 364     def check_sys_deps(self, needs_http):
 365         """If the port needs to do some runtime checks to ensure that the
 366         tests can be run successfully, it should override this routine.
 367         This step can be skipped with --nocheck-sys-deps.
 368
 369         Returns whether the system is properly configured."""
 370         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
 371
 372         local_error = ScriptError()
 373
 374         def error_handler(script_error):
 375             local_error.exit_code = script_error.exit_code
 376
 377         output = self._executive.run_command(cmd, error_handler=error_handler)
 378         if local_error.exit_code:
 379             _log.error('System dependencies check failed.')
 380             _log.error('To override, invoke with --nocheck-sys-deps')
 381             _log.error('')
 382             _log.error(output)
 383             return test_run_results.SYS_DEPS_EXIT_STATUS
 384         return test_run_results.OK_EXIT_STATUS
 385
 386     def check_image_diff(self, override_step=None, logging=True):
 387         """This routine is used to check whether image_diff binary exists."""
 388         image_diff_path = self._path_to_image_diff()
 389         if not self._filesystem.exists(image_diff_path):
 390             _log.error("image_diff was not found at %s" % image_diff_path)
 391             return False
 392         return True
 393
 394     def check_pretty_patch(self, logging=True):
 395         """Checks whether we can use the PrettyPatch ruby script."""
 396         try:
 397             _ = self._executive.run_command(['ruby', '--version'])
 398         except OSError, e:
 399             if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
 400                 if logging:
 401                     _log.warning("Ruby is not installed; can't generate pretty patches.")
 402                     _log.warning('')
 403                 return False
 404
 405         if not self._filesystem.exists(self._pretty_patch_path):
 406             if logging:
 407                 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path)
 408                 _log.warning('')
 409             return False
 410
 411         return True
 412
 413     def check_wdiff(self, logging=True):
 414         if not self._path_to_wdiff():
 415             # Don't need to log here since this is the port choosing not to use wdiff.
 416             return False
 417
 418         try:
 419             _ = self._executive.run_command([self._path_to_wdiff(), '--help'])
 420         except OSError:
 421             if logging:
 422                 message = self._wdiff_missing_message()
 423                 if message:
 424                     for line in message.splitlines():
 425                         _log.warning('    ' + line)
 426                         _log.warning('')
 427             return False
 428
 429         return True
 430
 431     def _wdiff_missing_message(self):
 432         return 'wdiff is not installed; please install it to generate word-by-word diffs.'
 433
 434     def check_httpd(self):
 435         if self._uses_apache():
 436             httpd_path = self._path_to_apache()
 437         else:
 438             httpd_path = self._path_to_lighttpd()
 439
 440         try:
 441             server_name = self._filesystem.basename(httpd_path)
 442             env = self.setup_environ_for_server(server_name)
 443             if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0:
 444                 _log.error("httpd seems broken. Cannot run http tests.")
 445                 return False
 446             return True
 447         except OSError:
 448             _log.error("No httpd found. Cannot run http tests.")
 449             return False
 450
 451     def do_text_results_differ(self, expected_text, actual_text):
 452         return expected_text != actual_text
 453
 454     def do_audio_results_differ(self, expected_audio, actual_audio):
 455         return expected_audio != actual_audio
 456
 457     def diff_image(self, expected_contents, actual_contents):
 458         """Compare two images and return a tuple of an image diff, and an error string.
 459
 460         If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff).
 461         """
 462         # If only one of them exists, return that one.
 463         if not actual_contents and not expected_contents:
 464             return (None, None)
 465         if not actual_contents:
 466             return (expected_contents, None)
 467         if not expected_contents:
 468             return (actual_contents, None)
 469
 470         tempdir = self._filesystem.mkdtemp()
 471
 472         expected_filename = self._filesystem.join(str(tempdir), "expected.png")
 473         self._filesystem.write_binary_file(expected_filename, expected_contents)
 474
 475         actual_filename = self._filesystem.join(str(tempdir), "actual.png")
 476         self._filesystem.write_binary_file(actual_filename, actual_contents)
 477
 478         diff_filename = self._filesystem.join(str(tempdir), "diff.png")
 479
 480         # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin.
 481         native_expected_filename = self._convert_path(expected_filename)
 482         native_actual_filename = self._convert_path(actual_filename)
 483         native_diff_filename = self._convert_path(diff_filename)
 484
 485         executable = self._path_to_image_diff()
 486         # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'.
 487         comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename]
 488
 489         result = None
 490         err_str = None
 491         try:
 492             exit_code = self._executive.run_command(comand, return_exit_code=True)
 493             if exit_code == 0:
 494                 # The images are the same.
 495                 result = None
 496             elif exit_code == 1:
 497                 result = self._filesystem.read_binary_file(native_diff_filename)
 498             else:
 499                 err_str = "image diff returned an exit code of %s" % exit_code
 500         except OSError, e:
 501             err_str = 'error running image diff: %s' % str(e)
 502         finally:
 503             self._filesystem.rmtree(str(tempdir))
 504
 505         return (result, err_str or None)
 506
 507     def diff_text(self, expected_text, actual_text, expected_filename, actual_filename):
 508         """Returns a string containing the diff of the two text strings
 509         in 'unified diff' format."""
 510
 511         # The filenames show up in the diff output, make sure they're
 512         # raw bytes and not unicode, so that they don't trigger join()
 513         # trying to decode the input.
 514         def to_raw_bytes(string_value):
 515             if isinstance(string_value, unicode):
 516                 return string_value.encode('utf-8')
 517             return string_value
 518         expected_filename = to_raw_bytes(expected_filename)
 519         actual_filename = to_raw_bytes(actual_filename)
 520         diff = difflib.unified_diff(expected_text.splitlines(True),
 521                                     actual_text.splitlines(True),
 522                                     expected_filename,
 523                                     actual_filename)
 524         return ''.join(diff)
 525
 526     def driver_name(self):
 527         if self.get_option('driver_name'):
 528             return self.get_option('driver_name')
 529         return self.CONTENT_SHELL_NAME
 530
 531     def expected_baselines_by_extension(self, test_name):
 532         """Returns a dict mapping baseline suffix to relative path for each baseline in
 533         a test. For reftests, it returns ".==" or ".!=" instead of the suffix."""
 534         # FIXME: The name similarity between this and expected_baselines() below, is unfortunate.
 535         # We should probably rename them both.
 536         baseline_dict = {}
 537         reference_files = self.reference_files(test_name)
 538         if reference_files:
 539             # FIXME: How should this handle more than one type of reftest?
 540             baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1])
 541
 542         for extension in self.baseline_extensions():
 543             path = self.expected_filename(test_name, extension, return_default=False)
 544             baseline_dict[extension] = self.relative_test_filename(path) if path else path
 545
 546         return baseline_dict
 547
 548     def baseline_extensions(self):
 549         """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'."""
 550         return ('.wav', '.txt', '.png')
 551
 552     def expected_baselines(self, test_name, suffix, all_baselines=False):
 553         """Given a test name, finds where the baseline results are located.
 554
 555         Args:
 556         test_name: name of test file (usually a relative path under LayoutTests/)
 557         suffix: file suffix of the expected results, including dot; e.g.
 558             '.txt' or '.png'.  This should not be None, but may be an empty
 559             string.
 560         all_baselines: If True, return an ordered list of all baseline paths
 561             for the given platform. If False, return only the first one.
 562         Returns
 563         a list of ( platform_dir, results_filename ), where
 564             platform_dir - abs path to the top of the results tree (or test
 565                 tree)
 566             results_filename - relative path from top of tree to the results
 567                 file
 568             (port.join() of the two gives you the full path to the file,
 569                 unless None was returned.)
 570         Return values will be in the format appropriate for the current
 571         platform (e.g., "\\" for path separators on Windows). If the results
 572         file is not found, then None will be returned for the directory,
 573         but the expected relative pathname will still be returned.
 574
 575         This routine is generic but lives here since it is used in
 576         conjunction with the other baseline and filename routines that are
 577         platform specific.
 578         """
 579         baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix
 580         baseline_search_path = self.baseline_search_path()
 581
 582         baselines = []
 583         for platform_dir in baseline_search_path:
 584             if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
 585                 baselines.append((platform_dir, baseline_filename))
 586
 587             if not all_baselines and baselines:
 588                 return baselines
 589
 590         # If it wasn't found in a platform directory, return the expected
 591         # result in the test directory, even if no such file actually exists.
 592         platform_dir = self.layout_tests_dir()
 593         if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
 594             baselines.append((platform_dir, baseline_filename))
 595
 596         if baselines:
 597             return baselines
 598
 599         return [(None, baseline_filename)]
 600
 601     def expected_filename(self, test_name, suffix, return_default=True):
 602         """Given a test name, returns an absolute path to its expected results.
 603
 604         If no expected results are found in any of the searched directories,
 605         the directory in which the test itself is located will be returned.
 606         The return value is in the format appropriate for the platform
 607         (e.g., "\\" for path separators on windows).
 608
 609         Args:
 610         test_name: name of test file (usually a relative path under LayoutTests/)
 611         suffix: file suffix of the expected results, including dot; e.g. '.txt'
 612             or '.png'.  This should not be None, but may be an empty string.
 613         platform: the most-specific directory name to use to build the
 614             search list of directories, e.g., 'win', or
 615             'chromium-cg-mac-leopard' (we follow the WebKit format)
 616         return_default: if True, returns the path to the generic expectation if nothing
 617             else is found; if False, returns None.
 618
 619         This routine is generic but is implemented here to live alongside
 620         the other baseline and filename manipulation routines.
 621         """
 622         # FIXME: The [0] here is very mysterious, as is the destructured return.
 623         platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0]
 624         if platform_dir:
 625             return self._filesystem.join(platform_dir, baseline_filename)
 626
 627         actual_test_name = self.lookup_virtual_test_base(test_name)
 628         if actual_test_name:
 629             return self.expected_filename(actual_test_name, suffix)
 630
 631         if return_default:
 632             return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
 633         return None
 634
 635     def expected_checksum(self, test_name):
 636         """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
 637         png_path = self.expected_filename(test_name, '.png')
 638
 639         if self._filesystem.exists(png_path):
 640             with self._filesystem.open_binary_file_for_reading(png_path) as filehandle:
 641                 return read_checksum_from_png.read_checksum(filehandle)
 642
 643         return None
 644
 645     def expected_image(self, test_name):
 646         """Returns the image we expect the test to produce."""
 647         baseline_path = self.expected_filename(test_name, '.png')
 648         if not self._filesystem.exists(baseline_path):
 649             return None
 650         return self._filesystem.read_binary_file(baseline_path)
 651
 652     def expected_audio(self, test_name):
 653         baseline_path = self.expected_filename(test_name, '.wav')
 654         if not self._filesystem.exists(baseline_path):
 655             return None
 656         return self._filesystem.read_binary_file(baseline_path)
 657
 658     def expected_text(self, test_name):
 659         """Returns the text output we expect the test to produce, or None
 660         if we don't expect there to be any text output.
 661         End-of-line characters are normalized to '\n'."""
 662         # FIXME: DRT output is actually utf-8, but since we don't decode the
 663         # output from DRT (instead treating it as a binary string), we read the
 664         # baselines as a binary string, too.
 665         baseline_path = self.expected_filename(test_name, '.txt')
 666         if not self._filesystem.exists(baseline_path):
 667             return None
 668         text = self._filesystem.read_binary_file(baseline_path)
 669         return text.replace("\r\n", "\n")
 670
 671     def _get_reftest_list(self, test_name):
 672         dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
 673         if dirname not in self._reftest_list:
 674             self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
 675         return self._reftest_list[dirname]
 676
 677     @staticmethod
 678     def _parse_reftest_list(filesystem, test_dirpath):
 679         reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
 680         if not filesystem.isfile(reftest_list_path):
 681             return None
 682         reftest_list_file = filesystem.read_text_file(reftest_list_path)
 683
 684         parsed_list = {}
 685         for line in reftest_list_file.split('\n'):
 686             line = re.sub('#.+$', '', line)
 687             split_line = line.split()
 688             if len(split_line) == 4:
 689                 # FIXME: Probably one of mozilla's extensions in the reftest.list format. Do we need to support this?
 690                 _log.warning("unsupported reftest.list line '%s' in %s" % (line, reftest_list_path))
 691                 continue
 692             if len(split_line) < 3:
 693                 continue
 694             expectation_type, test_file, ref_file = split_line
 695             parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
 696         return parsed_list
 697
 698     def reference_files(self, test_name):
 699         """Return a list of expectation (== or !=) and filename pairs"""
 700
 701         reftest_list = self._get_reftest_list(test_name)
 702         if not reftest_list:
 703             reftest_list = []
 704             for expectation, prefix in (('==', ''), ('!=', '-mismatch')):
 705                 for extention in Port._supported_file_extensions:
 706                     path = self.expected_filename(test_name, prefix + extention)
 707                     if self._filesystem.exists(path):
 708                         reftest_list.append((expectation, path))
 709             return reftest_list
 710
 711         return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), [])  # pylint: disable=E1103
 712
 713     def tests(self, paths):
 714         """Return the list of tests found matching paths."""
 715         tests = self._real_tests(paths)
 716         tests.extend(self._virtual_tests(paths, self.populated_virtual_test_suites()))
 717         return tests
 718
 719     def _real_tests(self, paths):
 720         # When collecting test cases, skip these directories
 721         skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'script-tests', 'reference', 'reftest'])
 722         files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key)
 723         return [self.relative_test_filename(f) for f in files]
 724
 725     # When collecting test cases, we include any file with these extensions.
 726     _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl',
 727                                       '.htm', '.php', '.svg', '.mht'])
 728
 729     @staticmethod
 730     # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
 731     def is_reference_html_file(filesystem, dirname, filename):
 732         if filename.startswith('ref-') or filename.startswith('notref-'):
 733             return True
 734         filename_wihout_ext, unused = filesystem.splitext(filename)
 735         for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
 736             if filename_wihout_ext.endswith(suffix):
 737                 return True
 738         return False
 739
 740     @staticmethod
 741     def _has_supported_extension(filesystem, filename):
 742         """Return true if filename is one of the file extensions we want to run a test on."""
 743         extension = filesystem.splitext(filename)[1]
 744         return extension in Port._supported_file_extensions
 745
 746     @staticmethod
 747     def is_test_file(filesystem, dirname, filename):
 748         return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
 749
 750     ALL_TEST_TYPES = ['audio', 'harness', 'pixel', 'ref', 'text', 'unknown']
 751
 752     def test_type(self, test_name):
 753         fs = self._filesystem
 754         if fs.exists(self.expected_filename(test_name, '.png')):
 755             return 'pixel'
 756         if fs.exists(self.expected_filename(test_name, '.wav')):
 757             return 'audio'
 758         if self.reference_files(test_name):
 759             return 'ref'
 760         txt = self.expected_text(test_name)
 761         if txt:
 762             if 'layer at (0,0) size 800x600' in txt:
 763                 return 'pixel'
 764             for line in txt.splitlines():
 765                 if line.startswith('FAIL') or line.startswith('TIMEOUT') or line.startswith('PASS'):
 766                     return 'harness'
 767             return 'text'
 768         return 'unknown'
 769
 770     def test_key(self, test_name):
 771         """Turns a test name into a list with two sublists, the natural key of the
 772         dirname, and the natural key of the basename.
 773
 774         This can be used when sorting paths so that files in a directory.
 775         directory are kept together rather than being mixed in with files in
 776         subdirectories."""
 777         dirname, basename = self.split_test(test_name)
 778         return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename))
 779
 780     def _natural_sort_key(self, string_to_split):
 781         """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
 782
 783         This can be used to implement "natural sort" order. See:
 784         http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
 785         http://nedbatchelder.com/blog/200712.html#e20071211T054956
 786         """
 787         def tryint(val):
 788             try:
 789                 return int(val)
 790             except ValueError:
 791                 return val
 792
 793         return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
 794
 795     def test_dirs(self):
 796         """Returns the list of top-level test directories."""
 797         layout_tests_dir = self.layout_tests_dir()
 798         return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
 799                       self._filesystem.listdir(layout_tests_dir))
 800
 801     @memoized
 802     def test_isfile(self, test_name):
 803         """Return True if the test name refers to a directory of tests."""
 804         # Used by test_expectations.py to apply rules to whole directories.
 805         if self._filesystem.isfile(self.abspath_for_test(test_name)):
 806             return True
 807         base = self.lookup_virtual_test_base(test_name)
 808         return base and self._filesystem.isfile(self.abspath_for_test(base))
 809
 810     @memoized
 811     def test_isdir(self, test_name):
 812         """Return True if the test name refers to a directory of tests."""
 813         # Used by test_expectations.py to apply rules to whole directories.
 814         if self._filesystem.isdir(self.abspath_for_test(test_name)):
 815             return True
 816         base = self.lookup_virtual_test_base(test_name)
 817         return base and self._filesystem.isdir(self.abspath_for_test(base))
 818
 819     @memoized
 820     def test_exists(self, test_name):
 821         """Return True if the test name refers to an existing test or baseline."""
 822         # Used by test_expectations.py to determine if an entry refers to a
 823         # valid test and by printing.py to determine if baselines exist.
 824         return self.test_isfile(test_name) or self.test_isdir(test_name)
 825
 826     def split_test(self, test_name):
 827         """Splits a test name into the 'directory' part and the 'basename' part."""
 828         index = test_name.rfind(self.TEST_PATH_SEPARATOR)
 829         if index < 1:
 830             return ('', test_name)
 831         return (test_name[0:index], test_name[index:])
 832
 833     def normalize_test_name(self, test_name):
 834         """Returns a normalized version of the test name or test directory."""
 835         if test_name.endswith('/'):
 836             return test_name
 837         if self.test_isdir(test_name):
 838             return test_name + '/'
 839         return test_name
 840
 841     def driver_cmd_line(self):
 842         """Prints the DRT command line that will be used."""
 843         driver = self.create_driver(0)
 844         return driver.cmd_line(self.get_option('pixel_tests'), [])
 845
 846     def update_baseline(self, baseline_path, data):
 847         """Updates the baseline for a test.
 848
 849         Args:
 850             baseline_path: the actual path to use for baseline, not the path to
 851               the test. This function is used to update either generic or
 852               platform-specific baselines, but we can't infer which here.
 853             data: contents of the baseline.
 854         """
 855         self._filesystem.write_binary_file(baseline_path, data)
 856
 857     # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected).
 858     def webkit_base(self):
 859         return self._webkit_finder.webkit_base()
 860
 861     def path_from_webkit_base(self, *comps):
 862         return self._webkit_finder.path_from_webkit_base(*comps)
 863
 864     def path_from_chromium_base(self, *comps):
 865         return self._webkit_finder.path_from_chromium_base(*comps)
 866
 867     def path_to_script(self, script_name):
 868         return self._webkit_finder.path_to_script(script_name)
 869
 870     def layout_tests_dir(self):
 871         return self._webkit_finder.layout_tests_dir()
 872
 873     def perf_tests_dir(self):
 874         return self._webkit_finder.perf_tests_dir()
 875
 876     def skipped_layout_tests(self, test_list):
 877         """Returns tests skipped outside of the TestExpectations files."""
 878         return set(self._skipped_tests_for_unsupported_features(test_list))
 879
 880     def _tests_from_skipped_file_contents(self, skipped_file_contents):
 881         tests_to_skip = []
 882         for line in skipped_file_contents.split('\n'):
 883             line = line.strip()
 884             line = line.rstrip('/')  # Best to normalize directory names to not include the trailing slash.
 885             if line.startswith('#') or not len(line):
 886                 continue
 887             tests_to_skip.append(line)
 888         return tests_to_skip
 889
 890     def _expectations_from_skipped_files(self, skipped_file_paths):
 891         tests_to_skip = []
 892         for search_path in skipped_file_paths:
 893             filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped")
 894             if not self._filesystem.exists(filename):
 895                 _log.debug("Skipped does not exist: %s" % filename)
 896                 continue
 897             _log.debug("Using Skipped file: %s" % filename)
 898             skipped_file_contents = self._filesystem.read_text_file(filename)
 899             tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
 900         return tests_to_skip
 901
 902     @memoized
 903     def skipped_perf_tests(self):
 904         return self._expectations_from_skipped_files([self.perf_tests_dir()])
 905
 906     def skips_perf_test(self, test_name):
 907         for test_or_category in self.skipped_perf_tests():
 908             if test_or_category == test_name:
 909                 return True
 910             category = self._filesystem.join(self.perf_tests_dir(), test_or_category)
 911             if self._filesystem.isdir(category) and test_name.startswith(test_or_category):
 912                 return True
 913         return False
 914
 915     def is_chromium(self):
 916         return True
 917
 918     def name(self):
 919         """Returns a name that uniquely identifies this particular type of port
 920         (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed
 921         to factory.get() to instantiate the port."""
 922         return self._name
 923
 924     def operating_system(self):
 925         # Subclasses should override this default implementation.
 926         return 'mac'
 927
 928     def version(self):
 929         """Returns a string indicating the version of a given platform, e.g.
 930         'leopard' or 'xp'.
 931
 932         This is used to help identify the exact port when parsing test
 933         expectations, determining search paths, and logging information."""
 934         return self._version
 935
 936     def architecture(self):
 937         return self._architecture
 938
 939     def get_option(self, name, default_value=None):
 940         return getattr(self._options, name, default_value)
 941
 942     def set_option_default(self, name, default_value):
 943         return self._options.ensure_value(name, default_value)
 944
 945     @memoized
 946     def path_to_generic_test_expectations_file(self):
 947         return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations')
 948
 949     def relative_test_filename(self, filename):
 950         """Returns a test_name a relative unix-style path for a filename under the LayoutTests
 951         directory. Ports may legitimately return abspaths here if no relpath makes sense."""
 952         # Ports that run on windows need to override this method to deal with
 953         # filenames with backslashes in them.
 954         if filename.startswith(self.layout_tests_dir()):
 955             return self.host.filesystem.relpath(filename, self.layout_tests_dir())
 956         else:
 957             return self.host.filesystem.abspath(filename)
 958
 959     @memoized
 960     def abspath_for_test(self, test_name):
 961         """Returns the full path to the file for a given test name. This is the
 962         inverse of relative_test_filename()."""
 963         return self._filesystem.join(self.layout_tests_dir(), test_name)
 964
 965     def results_directory(self):
 966         """Absolute path to the place to store the test results (uses --results-directory)."""
 967         if not self._results_directory:
 968             option_val = self.get_option('results_directory') or self.default_results_directory()
 969             self._results_directory = self._filesystem.abspath(option_val)
 970         return self._results_directory
 971
 972     def perf_results_directory(self):
 973         return self._build_path()
 974
 975     def default_results_directory(self):
 976         """Absolute path to the default place to store the test results."""
 977         try:
 978             return self.path_from_chromium_base('webkit', self.get_option('configuration'), 'layout-test-results')
 979         except AssertionError:
 980             return self._build_path('layout-test-results')
 981
 982     def setup_test_run(self):
 983         """Perform port-specific work at the beginning of a test run."""
 984         # Delete the disk cache if any to ensure a clean test run.
 985         dump_render_tree_binary_path = self._path_to_driver()
 986         cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
 987         cachedir = self._filesystem.join(cachedir, "cache")
 988         if self._filesystem.exists(cachedir):
 989             self._filesystem.rmtree(cachedir)
 990
 991         if self._dump_reader:
 992             self._filesystem.maybe_make_directory(self._dump_reader.crash_dumps_directory())
 993
 994     def num_workers(self, requested_num_workers):
 995         """Returns the number of available workers (possibly less than the number requested)."""
 996         return requested_num_workers
 997
 998     def clean_up_test_run(self):
 999         """Perform port-specific work at the end of a test run."""
1000         if self._image_differ:
1001             self._image_differ.stop()
1002             self._image_differ = None
1003
1004     # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable.
1005     def _value_or_default_from_environ(self, name, default=None):
1006         if name in os.environ:
1007             return os.environ[name]
1008         return default
1009
1010     def _copy_value_from_environ_if_set(self, clean_env, name):
1011         if name in os.environ:
1012             clean_env[name] = os.environ[name]
1013
1014     def setup_environ_for_server(self, server_name=None):
1015         # We intentionally copy only a subset of os.environ when
1016         # launching subprocesses to ensure consistent test results.
1017         clean_env = {
1018             'LOCAL_RESOURCE_ROOT': self.layout_tests_dir(),  # FIXME: Is this used?
1019         }
1020         variables_to_copy = [
1021             'WEBKIT_TESTFONTS',  # FIXME: Is this still used?
1022             'WEBKITOUTPUTDIR',   # FIXME: Is this still used?
1023             'CHROME_DEVEL_SANDBOX',
1024             'CHROME_IPC_LOGGING',
1025             'ASAN_OPTIONS',
1026             'VALGRIND_LIB',
1027             'VALGRIND_LIB_INNER',
1028         ]
1029         if self.host.platform.is_linux() or self.host.platform.is_freebsd():
1030             variables_to_copy += [
1031                 'XAUTHORITY',
1032                 'HOME',
1033                 'LANG',
1034                 'LD_LIBRARY_PATH',
1035                 'DBUS_SESSION_BUS_ADDRESS',
1036                 'XDG_DATA_DIRS',
1037             ]
1038             clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1')
1039         if self.host.platform.is_mac():
1040             clean_env['DYLD_LIBRARY_PATH'] = self._build_path()
1041             clean_env['DYLD_FRAMEWORK_PATH'] = self._build_path()
1042             variables_to_copy += [
1043                 'HOME',
1044             ]
1045         if self.host.platform.is_win():
1046             variables_to_copy += [
1047                 'PATH',
1048                 'GYP_DEFINES',  # Required to locate win sdk.
1049             ]
1050         if self.host.platform.is_cygwin():
1051             variables_to_copy += [
1052                 'HOMEDRIVE',
1053                 'HOMEPATH',
1054                 '_NT_SYMBOL_PATH',
1055             ]
1056
1057         for variable in variables_to_copy:
1058             self._copy_value_from_environ_if_set(clean_env, variable)
1059
1060         for string_variable in self.get_option('additional_env_var', []):
1061             [name, value] = string_variable.split('=', 1)
1062             clean_env[name] = value
1063
1064         return clean_env
1065
1066     def show_results_html_file(self, results_filename):
1067         """This routine should display the HTML file pointed at by
1068         results_filename in a users' browser."""
1069         return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename))
1070
1071     def create_driver(self, worker_number, no_timeout=False):
1072         """Return a newly created Driver subclass for starting/stopping the test driver."""
1073         return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout)
1074
1075     def start_helper(self):
1076         """If a port needs to reconfigure graphics settings or do other
1077         things to ensure a known test configuration, it should override this
1078         method."""
1079         helper_path = self._path_to_helper()
1080         if helper_path:
1081             _log.debug("Starting layout helper %s" % helper_path)
1082             # Note: Not thread safe: http://bugs.python.org/issue2320
1083             self._helper = self._executive.popen([helper_path],
1084                 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
1085             is_ready = self._helper.stdout.readline()
1086             if not is_ready.startswith('ready'):
1087                 _log.error("layout_test_helper failed to be ready")
1088
1089     def requires_http_server(self):
1090         """Does the port require an HTTP server for running tests? This could
1091         be the case when the tests aren't run on the host platform."""
1092         return False
1093
1094     def start_http_server(self, additional_dirs=None, number_of_servers=None):
1095         """Start a web server. Raise an error if it can't start or is already running.
1096
1097         Ports can stub this out if they don't need a web server to be running."""
1098         assert not self._http_server, 'Already running an http server.'
1099
1100         if self._uses_apache():
1101             server = apache_http_server.LayoutTestApacheHttpd(self, self.results_directory(), additional_dirs=additional_dirs, number_of_servers=number_of_servers)
1102         else:
1103             server = http_server.Lighttpd(self, self.results_directory(), additional_dirs=additional_dirs, number_of_servers=number_of_servers)
1104
1105         server.start()
1106         self._http_server = server
1107
1108     def start_websocket_server(self):
1109         """Start a web server. Raise an error if it can't start or is already running.
1110
1111         Ports can stub this out if they don't need a websocket server to be running."""
1112         assert not self._websocket_server, 'Already running a websocket server.'
1113
1114         server = websocket_server.PyWebSocket(self, self.results_directory())
1115         server.start()
1116         self._websocket_server = server
1117
1118     def http_server_supports_ipv6(self):
1119         # Cygwin is the only platform to still use Apache 1.3, which only supports IPV4.
1120         # Once it moves to Apache 2, we can drop this method altogether.
1121         if self.host.platform.is_cygwin():
1122             return False
1123         return True
1124
1125     def stop_helper(self):
1126         """Shut down the test helper if it is running. Do nothing if
1127         it isn't, or it isn't available. If a port overrides start_helper()
1128         it must override this routine as well."""
1129         if self._helper:
1130             _log.debug("Stopping layout test helper")
1131             try:
1132                 self._helper.stdin.write("x\n")
1133                 self._helper.stdin.close()
1134                 self._helper.wait()
1135             except IOError, e:
1136                 pass
1137             finally:
1138                 self._helper = None
1139
1140     def stop_http_server(self):
1141         """Shut down the http server if it is running. Do nothing if it isn't."""
1142         if self._http_server:
1143             self._http_server.stop()
1144             self._http_server = None
1145
1146     def stop_websocket_server(self):
1147         """Shut down the websocket server if it is running. Do nothing if it isn't."""
1148         if self._websocket_server:
1149             self._websocket_server.stop()
1150             self._websocket_server = None
1151
1152     #
1153     # TEST EXPECTATION-RELATED METHODS
1154     #
1155
1156     def test_configuration(self):
1157         """Returns the current TestConfiguration for the port."""
1158         if not self._test_configuration:
1159             self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower())
1160         return self._test_configuration
1161
1162     # FIXME: Belongs on a Platform object.
1163     @memoized
1164     def all_test_configurations(self):
1165         """Returns a list of TestConfiguration instances, representing all available
1166         test configurations for this port."""
1167         return self._generate_all_test_configurations()
1168
1169     # FIXME: Belongs on a Platform object.
1170     def configuration_specifier_macros(self):
1171         """Ports may provide a way to abbreviate configuration specifiers to conveniently
1172         refer to them as one term or alias specific values to more generic ones. For example:
1173
1174         (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake.
1175         (lucid) -> linux  # Change specific name of the Linux distro to a more generic term.
1176
1177         Returns a dictionary, each key representing a macro term ('win', for example),
1178         and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7'])."""
1179         return self.CONFIGURATION_SPECIFIER_MACROS
1180
1181     def all_baseline_variants(self):
1182         """Returns a list of platform names sufficient to cover all the baselines.
1183
1184         The list should be sorted so that a later platform  will reuse
1185         an earlier platform's baselines if they are the same (e.g.,
1186         'snowleopard' should precede 'leopard')."""
1187         return self.ALL_BASELINE_VARIANTS
1188
1189     def _generate_all_test_configurations(self):
1190         """Returns a sequence of the TestConfigurations the port supports."""
1191         # By default, we assume we want to test every graphics type in
1192         # every configuration on every system.
1193         test_configurations = []
1194         for version, architecture in self.ALL_SYSTEMS:
1195             for build_type in self.ALL_BUILD_TYPES:
1196                 test_configurations.append(TestConfiguration(version, architecture, build_type))
1197         return test_configurations
1198
1199     try_builder_names = frozenset([
1200         'linux_layout',
1201         'mac_layout',
1202         'win_layout',
1203         'linux_layout_rel',
1204         'mac_layout_rel',
1205         'win_layout_rel',
1206     ])
1207
1208     def warn_if_bug_missing_in_test_expectations(self):
1209         return True
1210
1211     def _port_specific_expectations_files(self):
1212         paths = []
1213         paths.append(self.path_from_chromium_base('skia', 'skia_test_expectations.txt'))
1214         paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations_w3c.txt'))
1215         paths.append(self._filesystem.join(self.layout_tests_dir(), 'NeverFixTests'))
1216         paths.append(self._filesystem.join(self.layout_tests_dir(), 'StaleTestExpectations'))
1217         paths.append(self._filesystem.join(self.layout_tests_dir(), 'SlowTests'))
1218
1219         builder_name = self.get_option('builder_name', 'DUMMY_BUILDER_NAME')
1220         if builder_name == 'DUMMY_BUILDER_NAME' or '(deps)' in builder_name or builder_name in self.try_builder_names:
1221             paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations.txt'))
1222         return paths
1223
1224     def expectations_dict(self):
1225         """Returns an OrderedDict of name -> expectations strings.
1226         The names are expected to be (but not required to be) paths in the filesystem.
1227         If the name is a path, the file can be considered updatable for things like rebaselining,
1228         so don't use names that are paths if they're not paths.
1229         Generally speaking the ordering should be files in the filesystem in cascade order
1230         (TestExpectations followed by Skipped, if the port honors both formats),
1231         then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options."""
1232         # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict.
1233         expectations = OrderedDict()
1234
1235         for path in self.expectations_files():
1236             if self._filesystem.exists(path):
1237                 expectations[path] = self._filesystem.read_text_file(path)
1238
1239         for path in self.get_option('additional_expectations', []):
1240             expanded_path = self._filesystem.expanduser(path)
1241             if self._filesystem.exists(expanded_path):
1242                 _log.debug("reading additional_expectations from path '%s'" % path)
1243                 expectations[path] = self._filesystem.read_text_file(expanded_path)
1244             else:
1245                 _log.warning("additional_expectations path '%s' does not exist" % path)
1246         return expectations
1247
1248     def bot_expectations(self):
1249         if not self.get_option('ignore_flaky_tests'):
1250             return {}
1251
1252         full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name)
1253         builder_category = self.get_option('ignore_builder_category', 'layout')
1254         factory = BotTestExpectationsFactory()
1255         expectations = factory.expectations_for_port(full_port_name, builder_category)
1256
1257         if not expectations:
1258             return {}
1259
1260         ignore_mode = self.get_option('ignore_flaky_tests')
1261         if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky':
1262             return expectations.flakes_by_path(ignore_mode == 'very-flaky')
1263         if ignore_mode == 'unexpected':
1264             return expectations.unexpected_results_by_path()
1265         _log.warning("Unexpected ignore mode: '%s'." % ignore_mode)
1266         return {}
1267
1268     def expectations_files(self):
1269         return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files()
1270
1271     def repository_paths(self):
1272         """Returns a list of (repository_name, repository_path) tuples of its depending code base."""
1273         return [('blink', self.layout_tests_dir()),
1274                 ('chromium', self.path_from_chromium_base('build'))]
1275
1276     _WDIFF_DEL = '##WDIFF_DEL##'
1277     _WDIFF_ADD = '##WDIFF_ADD##'
1278     _WDIFF_END = '##WDIFF_END##'
1279
1280     def _format_wdiff_output_as_html(self, wdiff):
1281         wdiff = cgi.escape(wdiff)
1282         wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
1283         wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
1284         wdiff = wdiff.replace(self._WDIFF_END, "</span>")
1285         html = "<head><style>.del { background: #faa; } "
1286         html += ".add { background: #afa; }</style></head>"
1287         html += "<pre>%s</pre>" % wdiff
1288         return html
1289
1290     def _wdiff_command(self, actual_filename, expected_filename):
1291         executable = self._path_to_wdiff()
1292         return [executable,
1293                 "--start-delete=%s" % self._WDIFF_DEL,
1294                 "--end-delete=%s" % self._WDIFF_END,
1295                 "--start-insert=%s" % self._WDIFF_ADD,
1296                 "--end-insert=%s" % self._WDIFF_END,
1297                 actual_filename,
1298                 expected_filename]
1299
1300     @staticmethod
1301     def _handle_wdiff_error(script_error):
1302         # Exit 1 means the files differed, any other exit code is an error.
1303         if script_error.exit_code != 1:
1304             raise script_error
1305
1306     def _run_wdiff(self, actual_filename, expected_filename):
1307         """Runs wdiff and may throw exceptions.
1308         This is mostly a hook for unit testing."""
1309         # Diffs are treated as binary as they may include multiple files
1310         # with conflicting encodings.  Thus we do not decode the output.
1311         command = self._wdiff_command(actual_filename, expected_filename)
1312         wdiff = self._executive.run_command(command, decode_output=False,
1313             error_handler=self._handle_wdiff_error)
1314         return self._format_wdiff_output_as_html(wdiff)
1315
1316     _wdiff_error_html = "Failed to run wdiff, see error log."
1317
1318     def wdiff_text(self, actual_filename, expected_filename):
1319         """Returns a string of HTML indicating the word-level diff of the
1320         contents of the two filenames. Returns an empty string if word-level
1321         diffing isn't available."""
1322         if not self.wdiff_available():
1323             return ""
1324         try:
1325             # It's possible to raise a ScriptError we pass wdiff invalid paths.
1326             return self._run_wdiff(actual_filename, expected_filename)
1327         except OSError as e:
1328             if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
1329                 # Silently ignore cases where wdiff is missing.
1330                 self._wdiff_available = False
1331                 return ""
1332             raise
1333         except ScriptError as e:
1334             _log.error("Failed to run wdiff: %s" % e)
1335             self._wdiff_available = False
1336             return self._wdiff_error_html
1337
1338     # This is a class variable so we can test error output easily.
1339     _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
1340
1341     def pretty_patch_text(self, diff_path):
1342         if self._pretty_patch_available is None:
1343             self._pretty_patch_available = self.check_pretty_patch(logging=False)
1344         if not self._pretty_patch_available:
1345             return self._pretty_patch_error_html
1346         command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path),
1347                    self._pretty_patch_path, diff_path)
1348         try:
1349             # Diffs are treated as binary (we pass decode_output=False) as they
1350             # may contain multiple files of conflicting encodings.
1351             return self._executive.run_command(command, decode_output=False)
1352         except OSError, e:
1353             # If the system is missing ruby log the error and stop trying.
1354             self._pretty_patch_available = False
1355             _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
1356             return self._pretty_patch_error_html
1357         except ScriptError, e:
1358             # If ruby failed to run for some reason, log the command
1359             # output and stop trying.
1360             self._pretty_patch_available = False
1361             _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output()))
1362             return self._pretty_patch_error_html
1363
1364     def default_configuration(self):
1365         return self._config.default_configuration()
1366
1367     def clobber_old_port_specific_results(self):
1368         pass
1369
1370     #
1371     # PROTECTED ROUTINES
1372     #
1373     # The routines below should only be called by routines in this class
1374     # or any of its subclasses.
1375     #
1376
1377     def _uses_apache(self):
1378         return True
1379
1380     # FIXME: This does not belong on the port object.
1381     @memoized
1382     def _path_to_apache(self):
1383         """Returns the full path to the apache binary.
1384
1385         This is needed only by ports that use the apache_http_server module."""
1386         raise NotImplementedError('Port._path_to_apache')
1387
1388     # FIXME: This belongs on some platform abstraction instead of Port.
1389     def _is_redhat_based(self):
1390         return self._filesystem.exists('/etc/redhat-release')
1391
1392     def _is_debian_based(self):
1393         return self._filesystem.exists('/etc/debian_version')
1394
1395     def _apache_version(self):
1396         config = self._executive.run_command([self._path_to_apache(), '-v'])
1397         return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config)
1398
1399     # We pass sys_platform into this method to make it easy to unit test.
1400     def _apache_config_file_name_for_platform(self, sys_platform):
1401         if sys_platform == 'cygwin':
1402             return 'cygwin-httpd.conf'  # CYGWIN is the only platform to still use Apache 1.3.
1403         if sys_platform.startswith('linux'):
1404             if self._is_redhat_based():
1405                 return 'fedora-httpd-' + self._apache_version() + '.conf'
1406             if self._is_debian_based():
1407                 return 'debian-httpd-' + self._apache_version() + '.conf'
1408         # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
1409         return "apache2-httpd.conf"
1410
1411     def _path_to_apache_config_file(self):
1412         """Returns the full path to the apache configuration file.
1413
1414         If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its
1415         contents will be used instead.
1416
1417         This is needed only by ports that use the apache_http_server module."""
1418         config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH')
1419         if config_file_from_env:
1420             if not self._filesystem.exists(config_file_from_env):
1421                 raise IOError('%s was not found on the system' % config_file_from_env)
1422             return config_file_from_env
1423
1424         config_file_name = self._apache_config_file_name_for_platform(sys.platform)
1425         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
1426
1427     def _path_to_driver(self, configuration=None):
1428         """Returns the full path to the test driver."""
1429         return self._build_path(self.driver_name())
1430
1431     def _path_to_webcore_library(self):
1432         """Returns the full path to a built copy of WebCore."""
1433         return None
1434
1435     def _path_to_helper(self):
1436         """Returns the full path to the layout_test_helper binary, which
1437         is used to help configure the system for the test run, or None
1438         if no helper is needed.
1439
1440         This is likely only used by start/stop_helper()."""
1441         return None
1442
1443     def _path_to_image_diff(self):
1444         """Returns the full path to the image_diff binary, or None if it is not available.
1445
1446         This is likely used only by diff_image()"""
1447         return self._build_path('image_diff')
1448
1449     def _path_to_lighttpd(self):
1450         """Returns the path to the LigHTTPd binary.
1451
1452         This is needed only by ports that use the http_server.py module."""
1453         raise NotImplementedError('Port._path_to_lighttpd')
1454
1455     def _path_to_lighttpd_modules(self):
1456         """Returns the path to the LigHTTPd modules directory.
1457
1458         This is needed only by ports that use the http_server.py module."""
1459         raise NotImplementedError('Port._path_to_lighttpd_modules')
1460
1461     def _path_to_lighttpd_php(self):
1462         """Returns the path to the LigHTTPd PHP executable.
1463
1464         This is needed only by ports that use the http_server.py module."""
1465         raise NotImplementedError('Port._path_to_lighttpd_php')
1466
1467     @memoized
1468     def _path_to_wdiff(self):
1469         """Returns the full path to the wdiff binary, or None if it is not available.
1470
1471         This is likely used only by wdiff_text()"""
1472         for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"):
1473             if self._filesystem.exists(path):
1474                 return path
1475         return None
1476
1477     def _webkit_baseline_path(self, platform):
1478         """Return the  full path to the top of the baseline tree for a
1479         given platform."""
1480         return self._filesystem.join(self.layout_tests_dir(), 'platform', platform)
1481
1482     def _driver_class(self):
1483         """Returns the port's driver implementation."""
1484         return driver.Driver
1485
1486     def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
1487         if stderr and 'AddressSanitizer' in stderr:
1488             # Running the AddressSanitizer take a lot of memory, so we need to
1489             # serialize access to it across all the concurrently running drivers.
1490
1491             # FIXME: investigate using LLVM_SYMBOLIZER_PATH here to reduce the overhead.
1492             asan_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
1493             if self._filesystem.exists(asan_filter_path):
1494                 output = self._executive.run_command(['flock', sys.executable, asan_filter_path], input=stderr, decode_output=False)
1495                 stderr = self._executive.run_command(['c++filt'], input=output, decode_output=False)
1496
1497         name_str = name or '<unknown process name>'
1498         pid_str = str(pid or '<unknown>')
1499         stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines()
1500         stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines()
1501         return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str,
1502             '\n'.join(('STDOUT: ' + l) for l in stdout_lines),
1503             '\n'.join(('STDERR: ' + l) for l in stderr_lines)))
1504
1505     def look_for_new_crash_logs(self, crashed_processes, start_time):
1506         pass
1507
1508     def look_for_new_samples(self, unresponsive_processes, start_time):
1509         pass
1510
1511     def sample_process(self, name, pid):
1512         pass
1513
1514     def virtual_test_suites(self):
1515         return [
1516             VirtualTestSuite('gpu',
1517                              'fast/canvas',
1518                              ['--enable-accelerated-2d-canvas']),
1519             VirtualTestSuite('gpu',
1520                              'canvas/philip',
1521                              ['--enable-accelerated-2d-canvas']),
1522             VirtualTestSuite('threaded',
1523                              'compositing/visibility',
1524                              ['--enable-threaded-compositing']),
1525             VirtualTestSuite('threaded',
1526                              'compositing/webgl',
1527                              ['--enable-threaded-compositing']),
1528             VirtualTestSuite('gpu',
1529                              'fast/hidpi',
1530                              ['--force-compositing-mode']),
1531             VirtualTestSuite('softwarecompositing',
1532                              'compositing',
1533                              ['--enable-software-compositing', '--disable-gpu-compositing'],
1534                              use_legacy_naming=True),
1535             VirtualTestSuite('deferred',
1536                              'fast/images',
1537                              ['--enable-deferred-image-decoding', '--enable-per-tile-painting', '--force-compositing-mode']),
1538             VirtualTestSuite('deferred',
1539                              'inspector/timeline',
1540                              ['--enable-deferred-image-decoding', '--enable-per-tile-painting', '--force-compositing-mode']),
1541             VirtualTestSuite('gpu/compositedscrolling/overflow',
1542                              'compositing/overflow',
1543                              ['--enable-accelerated-overflow-scroll'],
1544                              use_legacy_naming=True),
1545             VirtualTestSuite('gpu/compositedscrolling/scrollbars',
1546                              'scrollbars',
1547                              ['--enable-accelerated-overflow-scroll'],
1548                              use_legacy_naming=True),
1549             VirtualTestSuite('threaded',
1550                              'animations',
1551                              ['--enable-threaded-compositing']),
1552             VirtualTestSuite('threaded',
1553                              'transitions',
1554                              ['--enable-threaded-compositing']),
1555             VirtualTestSuite('legacy-animations-engine',
1556                              'animations',
1557                              ['--disable-web-animations-css']),
1558             VirtualTestSuite('legacy-animations-engine',
1559                              'transitions',
1560                              ['--disable-web-animations-css']),
1561             VirtualTestSuite('stable',
1562                              'webexposed',
1563                              ['--stable-release-mode']),
1564             VirtualTestSuite('stable',
1565                              'media/stable',
1566                              ['--stable-release-mode']),
1567             VirtualTestSuite('android',
1568                              'fullscreen',
1569                              ['--force-compositing-mode', '--allow-webui-compositing', '--enable-threaded-compositing',
1570                               '--enable-fixed-position-compositing', '--enable-accelerated-overflow-scroll', '--enable-accelerated-scrollable-frames',
1571                               '--enable-composited-scrolling-for-frames', '--enable-gesture-tap-highlight', '--enable-pinch',
1572                               '--enable-overlay-fullscreen-video', '--enable-overlay-scrollbars', '--enable-overscroll-notifications',
1573                               '--enable-fixed-layout', '--enable-viewport', '--disable-canvas-aa',
1574                               '--disable-composited-antialiasing']),
1575             VirtualTestSuite('implsidepainting',
1576                              'inspector/timeline',
1577                              ['--enable-threaded-compositing', '--enable-impl-side-painting', '--force-compositing-mode']),
1578             VirtualTestSuite('fasttextautosizing',
1579                              'fast/text-autosizing',
1580                              ['--enable-fast-text-autosizing']),
1581             VirtualTestSuite('serviceworker',
1582                              'http/tests/serviceworker',
1583                              ['--enable-service-worker']),
1584         ]
1585
1586     @memoized
1587     def populated_virtual_test_suites(self):
1588         suites = self.virtual_test_suites()
1589
1590         # Sanity-check the suites to make sure they don't point to other suites.
1591         suite_dirs = [suite.name for suite in suites]
1592         for suite in suites:
1593             assert suite.base not in suite_dirs
1594
1595         for suite in suites:
1596             base_tests = self._real_tests([suite.base])
1597             suite.tests = {}
1598             for test in base_tests:
1599                 suite.tests[test.replace(suite.base, suite.name, 1)] = test
1600         return suites
1601
1602     def _virtual_tests(self, paths, suites):
1603         virtual_tests = list()
1604         for suite in suites:
1605             if paths:
1606                 for test in suite.tests:
1607                     if any(test.startswith(p) for p in paths):
1608                         virtual_tests.append(test)
1609             else:
1610                 virtual_tests.extend(suite.tests.keys())
1611         return virtual_tests
1612
1613     def is_virtual_test(self, test_name):
1614         return bool(self.lookup_virtual_suite(test_name))
1615
1616     def lookup_virtual_suite(self, test_name):
1617         for suite in self.populated_virtual_test_suites():
1618             if test_name.startswith(suite.name):
1619                 return suite
1620         return None
1621
1622     def lookup_virtual_test_base(self, test_name):
1623         suite = self.lookup_virtual_suite(test_name)
1624         if not suite:
1625             return None
1626         return test_name.replace(suite.name, suite.base, 1)
1627
1628     def lookup_virtual_test_args(self, test_name):
1629         for suite in self.populated_virtual_test_suites():
1630             if test_name.startswith(suite.name):
1631                 return suite.args
1632         return []
1633
1634     def should_run_as_pixel_test(self, test_input):
1635         if not self._options.pixel_tests:
1636             return False
1637         if self._options.pixel_test_directories:
1638             return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories)
1639         return True
1640
1641     def _modules_to_search_for_symbols(self):
1642         path = self._path_to_webcore_library()
1643         if path:
1644             return [path]
1645         return []
1646
1647     def _symbols_string(self):
1648         symbols = ''
1649         for path_to_module in self._modules_to_search_for_symbols():
1650             try:
1651                 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error)
1652             except OSError, e:
1653                 _log.warn("Failed to run nm: %s.  Can't determine supported features correctly." % e)
1654         return symbols
1655
1656     # Ports which use compile-time feature detection should define this method and return
1657     # a dictionary mapping from symbol substrings to possibly disabled test directories.
1658     # When the symbol substrings are not matched, the directories will be skipped.
1659     # If ports don't ever enable certain features, then those directories can just be
1660     # in the Skipped list instead of compile-time-checked here.
1661     def _missing_symbol_to_skipped_tests(self):
1662         if self.PORT_HAS_AUDIO_CODECS_BUILT_IN:
1663             return {}
1664         else:
1665             return {
1666                 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"],
1667                 "ff_aac_decoder": ["webaudio/codec-tests/aac"],
1668             }
1669
1670     def _has_test_in_directories(self, directory_lists, test_list):
1671         if not test_list:
1672             return False
1673
1674         directories = itertools.chain.from_iterable(directory_lists)
1675         for directory, test in itertools.product(directories, test_list):
1676             if test.startswith(directory):
1677                 return True
1678         return False
1679
1680     def _skipped_tests_for_unsupported_features(self, test_list):
1681         # Only check the symbols of there are tests in the test_list that might get skipped.
1682         # This is a performance optimization to avoid the calling nm.
1683         # Runtime feature detection not supported, fallback to static detection:
1684         # Disable any tests for symbols missing from the executable or libraries.
1685         if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
1686             symbols_string = self._symbols_string()
1687             if symbols_string is not None:
1688                 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], [])
1689         return []
1690
1691     def _convert_path(self, path):
1692         """Handles filename conversion for subprocess command line args."""
1693         # See note above in diff_image() for why we need this.
1694         if sys.platform == 'cygwin':
1695             return cygpath(path)
1696         return path
1697
1698     def _build_path(self, *comps):
1699         return self._build_path_with_configuration(None, *comps)
1700
1701     def _build_path_with_configuration(self, configuration, *comps):
1702         # Note that we don't do the option caching that the
1703         # base class does, because finding the right directory is relatively
1704         # fast.
1705         configuration = configuration or self.get_option('configuration')
1706         return self._static_build_path(self._filesystem, self.get_option('build_directory'),
1707             self.path_from_chromium_base(), configuration, comps)
1708
1709     def _check_driver_build_up_to_date(self, configuration):
1710         if configuration in ('Debug', 'Release'):
1711             try:
1712                 debug_path = self._path_to_driver('Debug')
1713                 release_path = self._path_to_driver('Release')
1714
1715                 debug_mtime = self._filesystem.mtime(debug_path)
1716                 release_mtime = self._filesystem.mtime(release_path)
1717
1718                 if (debug_mtime > release_mtime and configuration == 'Release' or
1719                     release_mtime > debug_mtime and configuration == 'Debug'):
1720                     most_recent_binary = 'Release' if configuration == 'Debug' else 'Debug'
1721                     _log.warning('You are running the %s binary. However the %s binary appears to be more recent. '
1722                                  'Please pass --%s.', configuration, most_recent_binary, most_recent_binary.lower())
1723                     _log.warning('')
1724             # This will fail if we don't have both a debug and release binary.
1725             # That's fine because, in this case, we must already be running the
1726             # most up-to-date one.
1727             except OSError:
1728                 pass
1729         return True
1730
1731     def _chromium_baseline_path(self, platform):
1732         if platform is None:
1733             platform = self.name()
1734         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
1735
1736 class VirtualTestSuite(object):
1737     def __init__(self, name, base, args, use_legacy_naming=False, tests=None):
1738         if use_legacy_naming:
1739             self.name = 'virtual/' + name
1740         else:
1741             if name.find('/') != -1:
1742                 _log.error("Virtual test suites names cannot contain /'s: %s" % name)
1743                 return
1744             self.name = 'virtual/' + name + '/' + base
1745         self.base = base
1746         self.args = args
1747         self.tests = tests or set()
1748
1749     def __repr__(self):
1750         return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)