1 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
13 # * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 """Abstract base class of Port-specific entry points for the layout tests
30 test infrastructure (the Port and Driver classes)."""
44 from collections import OrderedDict
46 # Needed for Python < 2.7
47 from webkitpy.thirdparty.ordered_dict import OrderedDict
50 from webkitpy.common import find_files
51 from webkitpy.common import read_checksum_from_png
52 from webkitpy.common.memoized import memoized
53 from webkitpy.common.system import path
54 from webkitpy.common.system.executive import ScriptError
55 from webkitpy.common.system.path import cygpath
56 from webkitpy.common.system.systemhost import SystemHost
57 from webkitpy.common.webkit_finder import WebKitFinder
58 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
59 from webkitpy.layout_tests.models import test_run_results
60 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
61 from webkitpy.layout_tests.port import config as port_config
62 from webkitpy.layout_tests.port import driver
63 from webkitpy.layout_tests.port import server_process
64 from webkitpy.layout_tests.port.factory import PortFactory
65 from webkitpy.layout_tests.servers import apache_http_server
66 from webkitpy.layout_tests.servers import http_server
67 from webkitpy.layout_tests.servers import websocket_server
69 _log = logging.getLogger(__name__)
72 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports.
74 """Abstract class for Port-specific hooks for the layout_test package."""
76 # Subclasses override this. This should indicate the basic implementation
77 # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?)
78 # one unique value per class.
80 # FIXME: We should probably rename this to something like 'implementation_name'.
83 # Test names resemble unix relative paths, and use '/' as a directory separator.
84 TEST_PATH_SEPARATOR = '/'
86 ALL_BUILD_TYPES = ('debug', 'release')
88 CONTENT_SHELL_NAME = 'content_shell'
90 # True if the port as aac and mp3 codecs built in.
91 PORT_HAS_AUDIO_CODECS_BUILT_IN = False
94 ('snowleopard', 'x86'),
97 # FIXME: We treat Retina (High-DPI) devices as if they are running
98 # a different operating system version. This isn't accurate, but will work until
99 # we need to test and support baselines across multiple O/S versions.
102 ('mountainlion', 'x86'),
103 ('mavericks', 'x86'),
108 # FIXME: Technically this should be 'arm', but adding a third architecture type breaks TestConfigurationConverter.
109 # If we need this to be 'arm' in the future, then we first have to fix TestConfigurationConverter.
110 ('icecreamsandwich', 'x86'),
113 ALL_BASELINE_VARIANTS = [
114 'mac-mavericks', 'mac-mountainlion', 'mac-retina', 'mac-lion', 'mac-snowleopard',
115 'win-win7', 'win-xp',
116 'linux-x86_64', 'linux-x86',
119 CONFIGURATION_SPECIFIER_MACROS = {
120 'mac': ['snowleopard', 'lion', 'retina', 'mountainlion', 'mavericks'],
121 'win': ['xp', 'win7'],
123 'android': ['icecreamsandwich'],
126 DEFAULT_BUILD_DIRECTORIES = ('out',)
128 # overridden in subclasses.
131 SUPPORTED_VERSIONS = []
134 def latest_platform_fallback_path(cls):
135 return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]]
138 def _static_build_path(cls, filesystem, build_directory, chromium_base, configuration, comps):
140 return filesystem.join(build_directory, configuration, *comps)
143 for directory in cls.DEFAULT_BUILD_DIRECTORIES:
144 base_dir = filesystem.join(chromium_base, directory, configuration)
145 path = filesystem.join(base_dir, *comps)
146 if filesystem.exists(path):
147 hits.append((filesystem.mtime(path), path))
150 hits.sort(reverse=True)
151 return hits[0][1] # Return the newest file found.
153 # We have to default to something, so pick the last one.
154 return filesystem.join(base_dir, *comps)
157 def determine_full_port_name(cls, host, options, port_name):
158 """Return a fully-specified port name that can be used to construct objects."""
159 # Subclasses will usually override this.
160 assert port_name.startswith(cls.port_name)
163 def __init__(self, host, port_name, options=None, **kwargs):
165 # This value may be different from cls.port_name by having version modifiers
166 # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2').
167 self._name = port_name
169 # These are default values that should be overridden in a subclasses.
171 self._architecture = 'x86'
173 # FIXME: Ideally we'd have a package-wide way to get a
174 # well-formed options object that had all of the necessary
175 # options defined on it.
176 self._options = options or optparse.Values()
179 self._executive = host.executive
180 self._filesystem = host.filesystem
181 self._webkit_finder = WebKitFinder(host.filesystem)
182 self._config = port_config.Config(self._executive, self._filesystem, self.port_name)
185 self._http_server = None
186 self._websocket_server = None
187 self._image_differ = None
188 self._server_process_constructor = server_process.ServerProcess # overridable for testing
189 self._http_lock = None # FIXME: Why does this live on the port object?
190 self._dump_reader = None
192 # Python's Popen has a bug that causes any pipes opened to a
193 # process that can't be executed to be leaked. Since this
194 # code is specifically designed to tolerate exec failures
195 # to gracefully handle cases where wdiff is not installed,
196 # the bug results in a massive file descriptor leak. As a
197 # workaround, if an exec failure is ever experienced for
198 # wdiff, assume it's not available. This will leak one
199 # file descriptor but that's better than leaking each time
200 # wdiff would be run.
202 # http://mail.python.org/pipermail/python-list/
203 # 2008-August/505753.html
204 # http://bugs.python.org/issue3210
205 self._wdiff_available = None
207 # FIXME: prettypatch.py knows this path, why is it copied here?
208 self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb")
209 self._pretty_patch_available = None
211 if not hasattr(options, 'configuration') or not options.configuration:
212 self.set_option_default('configuration', self.default_configuration())
213 self._test_configuration = None
214 self._reftest_list = {}
215 self._results_directory = None
217 def buildbot_archives_baselines(self):
220 def additional_drt_flag(self):
221 if self.driver_name() == self.CONTENT_SHELL_NAME:
222 return ['--dump-render-tree']
225 def supports_per_test_timeout(self):
228 def default_pixel_tests(self):
231 def default_smoke_test_only(self):
234 def default_timeout_ms(self):
235 timeout_ms = 6 * 1000
236 if self.get_option('configuration') == 'Debug':
237 # Debug is usually 2x-3x slower than Release.
238 return 3 * timeout_ms
241 def driver_stop_timeout(self):
242 """ Returns the amount of time in seconds to wait before killing the process in driver.stop()."""
243 # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as
244 # well (for things like ASAN, Valgrind, etc.)
245 return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms()
247 def wdiff_available(self):
248 if self._wdiff_available is None:
249 self._wdiff_available = self.check_wdiff(logging=False)
250 return self._wdiff_available
252 def pretty_patch_available(self):
253 if self._pretty_patch_available is None:
254 self._pretty_patch_available = self.check_pretty_patch(logging=False)
255 return self._pretty_patch_available
257 def default_child_processes(self):
258 """Return the number of drivers to use for this port."""
259 return self._executive.cpu_count()
261 def default_max_locked_shards(self):
262 """Return the number of "locked" shards to run in parallel (like the http tests)."""
263 max_locked_shards = int(self.default_child_processes()) / 4
264 if not max_locked_shards:
266 return max_locked_shards
268 def baseline_path(self):
269 """Return the absolute path to the directory to store new baselines in for this port."""
270 # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
271 return self.baseline_version_dir()
273 def baseline_platform_dir(self):
274 """Return the absolute path to the default (version-independent) platform-specific results."""
275 return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
277 def baseline_version_dir(self):
278 """Return the absolute path to the platform-and-version-specific results."""
279 baseline_search_paths = self.baseline_search_path()
280 return baseline_search_paths[0]
282 def virtual_baseline_search_path(self, test_name):
283 suite = self.lookup_virtual_suite(test_name)
286 return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()]
288 def baseline_search_path(self):
289 return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path()
291 def default_baseline_search_path(self):
292 """Return a list of absolute paths to directories to search under for
293 baselines. The directories are searched in order."""
294 return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version()])
297 def _compare_baseline(self):
298 factory = PortFactory(self.host)
299 target_port = self.get_option('compare_port')
301 return factory.get(target_port).default_baseline_search_path()
304 def _check_file_exists(self, path_to_file, file_description,
305 override_step=None, logging=True):
306 """Verify the file is present where expected or log an error.
309 file_name: The (human friendly) name or description of the file
310 you're looking for (e.g., "HTTP Server"). Used for error logging.
311 override_step: An optional string to be logged if the check fails.
312 logging: Whether or not log the error messages."""
313 if not self._filesystem.exists(path_to_file):
315 _log.error('Unable to find %s' % file_description)
316 _log.error(' at %s' % path_to_file)
318 _log.error(' %s' % override_step)
323 def check_build(self, needs_http, printer):
326 dump_render_tree_binary_path = self._path_to_driver()
327 result = self._check_file_exists(dump_render_tree_binary_path,
328 'test driver') and result
329 if not result and self.get_option('build'):
330 result = self._check_driver_build_up_to_date(
331 self.get_option('configuration'))
335 helper_path = self._path_to_helper()
337 result = self._check_file_exists(helper_path,
338 'layout test helper') and result
340 if self.get_option('pixel_tests'):
341 result = self.check_image_diff(
342 'To override, invoke with --no-pixel-tests') and result
344 # It's okay if pretty patch and wdiff aren't available, but we will at least log messages.
345 self._pretty_patch_available = self.check_pretty_patch()
346 self._wdiff_available = self.check_wdiff()
348 if self._dump_reader:
349 result = self._dump_reader.check_is_functional() and result
351 return test_run_results.OK_EXIT_STATUS if result else test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
353 def _check_driver(self):
354 driver_path = self._path_to_driver()
355 if not self._filesystem.exists(driver_path):
356 _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
360 def _check_port_build(self):
361 # Ports can override this method to do additional checks.
364 def check_sys_deps(self, needs_http):
365 """If the port needs to do some runtime checks to ensure that the
366 tests can be run successfully, it should override this routine.
367 This step can be skipped with --nocheck-sys-deps.
369 Returns whether the system is properly configured."""
370 cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
372 local_error = ScriptError()
374 def error_handler(script_error):
375 local_error.exit_code = script_error.exit_code
377 output = self._executive.run_command(cmd, error_handler=error_handler)
378 if local_error.exit_code:
379 _log.error('System dependencies check failed.')
380 _log.error('To override, invoke with --nocheck-sys-deps')
383 return test_run_results.SYS_DEPS_EXIT_STATUS
384 return test_run_results.OK_EXIT_STATUS
386 def check_image_diff(self, override_step=None, logging=True):
387 """This routine is used to check whether image_diff binary exists."""
388 image_diff_path = self._path_to_image_diff()
389 if not self._filesystem.exists(image_diff_path):
390 _log.error("image_diff was not found at %s" % image_diff_path)
394 def check_pretty_patch(self, logging=True):
395 """Checks whether we can use the PrettyPatch ruby script."""
397 _ = self._executive.run_command(['ruby', '--version'])
399 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
401 _log.warning("Ruby is not installed; can't generate pretty patches.")
405 if not self._filesystem.exists(self._pretty_patch_path):
407 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path)
413 def check_wdiff(self, logging=True):
414 if not self._path_to_wdiff():
415 # Don't need to log here since this is the port choosing not to use wdiff.
419 _ = self._executive.run_command([self._path_to_wdiff(), '--help'])
422 message = self._wdiff_missing_message()
424 for line in message.splitlines():
425 _log.warning(' ' + line)
431 def _wdiff_missing_message(self):
432 return 'wdiff is not installed; please install it to generate word-by-word diffs.'
434 def check_httpd(self):
435 if self._uses_apache():
436 httpd_path = self._path_to_apache()
438 httpd_path = self._path_to_lighttpd()
441 server_name = self._filesystem.basename(httpd_path)
442 env = self.setup_environ_for_server(server_name)
443 if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0:
444 _log.error("httpd seems broken. Cannot run http tests.")
448 _log.error("No httpd found. Cannot run http tests.")
451 def do_text_results_differ(self, expected_text, actual_text):
452 return expected_text != actual_text
454 def do_audio_results_differ(self, expected_audio, actual_audio):
455 return expected_audio != actual_audio
457 def diff_image(self, expected_contents, actual_contents):
458 """Compare two images and return a tuple of an image diff, and an error string.
460 If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff).
462 # If only one of them exists, return that one.
463 if not actual_contents and not expected_contents:
465 if not actual_contents:
466 return (expected_contents, None)
467 if not expected_contents:
468 return (actual_contents, None)
470 tempdir = self._filesystem.mkdtemp()
472 expected_filename = self._filesystem.join(str(tempdir), "expected.png")
473 self._filesystem.write_binary_file(expected_filename, expected_contents)
475 actual_filename = self._filesystem.join(str(tempdir), "actual.png")
476 self._filesystem.write_binary_file(actual_filename, actual_contents)
478 diff_filename = self._filesystem.join(str(tempdir), "diff.png")
480 # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin.
481 native_expected_filename = self._convert_path(expected_filename)
482 native_actual_filename = self._convert_path(actual_filename)
483 native_diff_filename = self._convert_path(diff_filename)
485 executable = self._path_to_image_diff()
486 # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'.
487 comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename]
492 exit_code = self._executive.run_command(comand, return_exit_code=True)
494 # The images are the same.
497 result = self._filesystem.read_binary_file(native_diff_filename)
499 err_str = "image diff returned an exit code of %s" % exit_code
501 err_str = 'error running image diff: %s' % str(e)
503 self._filesystem.rmtree(str(tempdir))
505 return (result, err_str or None)
507 def diff_text(self, expected_text, actual_text, expected_filename, actual_filename):
508 """Returns a string containing the diff of the two text strings
509 in 'unified diff' format."""
511 # The filenames show up in the diff output, make sure they're
512 # raw bytes and not unicode, so that they don't trigger join()
513 # trying to decode the input.
514 def to_raw_bytes(string_value):
515 if isinstance(string_value, unicode):
516 return string_value.encode('utf-8')
518 expected_filename = to_raw_bytes(expected_filename)
519 actual_filename = to_raw_bytes(actual_filename)
520 diff = difflib.unified_diff(expected_text.splitlines(True),
521 actual_text.splitlines(True),
526 def driver_name(self):
527 if self.get_option('driver_name'):
528 return self.get_option('driver_name')
529 return self.CONTENT_SHELL_NAME
531 def expected_baselines_by_extension(self, test_name):
532 """Returns a dict mapping baseline suffix to relative path for each baseline in
533 a test. For reftests, it returns ".==" or ".!=" instead of the suffix."""
534 # FIXME: The name similarity between this and expected_baselines() below, is unfortunate.
535 # We should probably rename them both.
537 reference_files = self.reference_files(test_name)
539 # FIXME: How should this handle more than one type of reftest?
540 baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1])
542 for extension in self.baseline_extensions():
543 path = self.expected_filename(test_name, extension, return_default=False)
544 baseline_dict[extension] = self.relative_test_filename(path) if path else path
548 def baseline_extensions(self):
549 """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'."""
550 return ('.wav', '.txt', '.png')
552 def expected_baselines(self, test_name, suffix, all_baselines=False):
553 """Given a test name, finds where the baseline results are located.
556 test_name: name of test file (usually a relative path under LayoutTests/)
557 suffix: file suffix of the expected results, including dot; e.g.
558 '.txt' or '.png'. This should not be None, but may be an empty
560 all_baselines: If True, return an ordered list of all baseline paths
561 for the given platform. If False, return only the first one.
563 a list of ( platform_dir, results_filename ), where
564 platform_dir - abs path to the top of the results tree (or test
566 results_filename - relative path from top of tree to the results
568 (port.join() of the two gives you the full path to the file,
569 unless None was returned.)
570 Return values will be in the format appropriate for the current
571 platform (e.g., "\\" for path separators on Windows). If the results
572 file is not found, then None will be returned for the directory,
573 but the expected relative pathname will still be returned.
575 This routine is generic but lives here since it is used in
576 conjunction with the other baseline and filename routines that are
579 baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix
580 baseline_search_path = self.baseline_search_path()
583 for platform_dir in baseline_search_path:
584 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
585 baselines.append((platform_dir, baseline_filename))
587 if not all_baselines and baselines:
590 # If it wasn't found in a platform directory, return the expected
591 # result in the test directory, even if no such file actually exists.
592 platform_dir = self.layout_tests_dir()
593 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
594 baselines.append((platform_dir, baseline_filename))
599 return [(None, baseline_filename)]
601 def expected_filename(self, test_name, suffix, return_default=True):
602 """Given a test name, returns an absolute path to its expected results.
604 If no expected results are found in any of the searched directories,
605 the directory in which the test itself is located will be returned.
606 The return value is in the format appropriate for the platform
607 (e.g., "\\" for path separators on windows).
610 test_name: name of test file (usually a relative path under LayoutTests/)
611 suffix: file suffix of the expected results, including dot; e.g. '.txt'
612 or '.png'. This should not be None, but may be an empty string.
613 platform: the most-specific directory name to use to build the
614 search list of directories, e.g., 'win', or
615 'chromium-cg-mac-leopard' (we follow the WebKit format)
616 return_default: if True, returns the path to the generic expectation if nothing
617 else is found; if False, returns None.
619 This routine is generic but is implemented here to live alongside
620 the other baseline and filename manipulation routines.
622 # FIXME: The [0] here is very mysterious, as is the destructured return.
623 platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0]
625 return self._filesystem.join(platform_dir, baseline_filename)
627 actual_test_name = self.lookup_virtual_test_base(test_name)
629 return self.expected_filename(actual_test_name, suffix)
632 return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
635 def expected_checksum(self, test_name):
636 """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
637 png_path = self.expected_filename(test_name, '.png')
639 if self._filesystem.exists(png_path):
640 with self._filesystem.open_binary_file_for_reading(png_path) as filehandle:
641 return read_checksum_from_png.read_checksum(filehandle)
645 def expected_image(self, test_name):
646 """Returns the image we expect the test to produce."""
647 baseline_path = self.expected_filename(test_name, '.png')
648 if not self._filesystem.exists(baseline_path):
650 return self._filesystem.read_binary_file(baseline_path)
652 def expected_audio(self, test_name):
653 baseline_path = self.expected_filename(test_name, '.wav')
654 if not self._filesystem.exists(baseline_path):
656 return self._filesystem.read_binary_file(baseline_path)
658 def expected_text(self, test_name):
659 """Returns the text output we expect the test to produce, or None
660 if we don't expect there to be any text output.
661 End-of-line characters are normalized to '\n'."""
662 # FIXME: DRT output is actually utf-8, but since we don't decode the
663 # output from DRT (instead treating it as a binary string), we read the
664 # baselines as a binary string, too.
665 baseline_path = self.expected_filename(test_name, '.txt')
666 if not self._filesystem.exists(baseline_path):
668 text = self._filesystem.read_binary_file(baseline_path)
669 return text.replace("\r\n", "\n")
671 def _get_reftest_list(self, test_name):
672 dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
673 if dirname not in self._reftest_list:
674 self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
675 return self._reftest_list[dirname]
678 def _parse_reftest_list(filesystem, test_dirpath):
679 reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
680 if not filesystem.isfile(reftest_list_path):
682 reftest_list_file = filesystem.read_text_file(reftest_list_path)
685 for line in reftest_list_file.split('\n'):
686 line = re.sub('#.+$', '', line)
687 split_line = line.split()
688 if len(split_line) == 4:
689 # FIXME: Probably one of mozilla's extensions in the reftest.list format. Do we need to support this?
690 _log.warning("unsupported reftest.list line '%s' in %s" % (line, reftest_list_path))
692 if len(split_line) < 3:
694 expectation_type, test_file, ref_file = split_line
695 parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
698 def reference_files(self, test_name):
699 """Return a list of expectation (== or !=) and filename pairs"""
701 reftest_list = self._get_reftest_list(test_name)
704 for expectation, prefix in (('==', ''), ('!=', '-mismatch')):
705 for extention in Port._supported_file_extensions:
706 path = self.expected_filename(test_name, prefix + extention)
707 if self._filesystem.exists(path):
708 reftest_list.append((expectation, path))
711 return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), []) # pylint: disable=E1103
713 def tests(self, paths):
714 """Return the list of tests found matching paths."""
715 tests = self._real_tests(paths)
716 tests.extend(self._virtual_tests(paths, self.populated_virtual_test_suites()))
719 def _real_tests(self, paths):
720 # When collecting test cases, skip these directories
721 skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'script-tests', 'reference', 'reftest'])
722 files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key)
723 return [self.relative_test_filename(f) for f in files]
725 # When collecting test cases, we include any file with these extensions.
726 _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl',
727 '.htm', '.php', '.svg', '.mht'])
730 # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
731 def is_reference_html_file(filesystem, dirname, filename):
732 if filename.startswith('ref-') or filename.startswith('notref-'):
734 filename_wihout_ext, unused = filesystem.splitext(filename)
735 for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
736 if filename_wihout_ext.endswith(suffix):
741 def _has_supported_extension(filesystem, filename):
742 """Return true if filename is one of the file extensions we want to run a test on."""
743 extension = filesystem.splitext(filename)[1]
744 return extension in Port._supported_file_extensions
747 def is_test_file(filesystem, dirname, filename):
748 return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
750 ALL_TEST_TYPES = ['audio', 'harness', 'pixel', 'ref', 'text', 'unknown']
752 def test_type(self, test_name):
753 fs = self._filesystem
754 if fs.exists(self.expected_filename(test_name, '.png')):
756 if fs.exists(self.expected_filename(test_name, '.wav')):
758 if self.reference_files(test_name):
760 txt = self.expected_text(test_name)
762 if 'layer at (0,0) size 800x600' in txt:
764 for line in txt.splitlines():
765 if line.startswith('FAIL') or line.startswith('TIMEOUT') or line.startswith('PASS'):
770 def test_key(self, test_name):
771 """Turns a test name into a list with two sublists, the natural key of the
772 dirname, and the natural key of the basename.
774 This can be used when sorting paths so that files in a directory.
775 directory are kept together rather than being mixed in with files in
777 dirname, basename = self.split_test(test_name)
778 return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename))
780 def _natural_sort_key(self, string_to_split):
781 """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
783 This can be used to implement "natural sort" order. See:
784 http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
785 http://nedbatchelder.com/blog/200712.html#e20071211T054956
793 return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
796 """Returns the list of top-level test directories."""
797 layout_tests_dir = self.layout_tests_dir()
798 return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
799 self._filesystem.listdir(layout_tests_dir))
802 def test_isfile(self, test_name):
803 """Return True if the test name refers to a directory of tests."""
804 # Used by test_expectations.py to apply rules to whole directories.
805 if self._filesystem.isfile(self.abspath_for_test(test_name)):
807 base = self.lookup_virtual_test_base(test_name)
808 return base and self._filesystem.isfile(self.abspath_for_test(base))
811 def test_isdir(self, test_name):
812 """Return True if the test name refers to a directory of tests."""
813 # Used by test_expectations.py to apply rules to whole directories.
814 if self._filesystem.isdir(self.abspath_for_test(test_name)):
816 base = self.lookup_virtual_test_base(test_name)
817 return base and self._filesystem.isdir(self.abspath_for_test(base))
820 def test_exists(self, test_name):
821 """Return True if the test name refers to an existing test or baseline."""
822 # Used by test_expectations.py to determine if an entry refers to a
823 # valid test and by printing.py to determine if baselines exist.
824 return self.test_isfile(test_name) or self.test_isdir(test_name)
826 def split_test(self, test_name):
827 """Splits a test name into the 'directory' part and the 'basename' part."""
828 index = test_name.rfind(self.TEST_PATH_SEPARATOR)
830 return ('', test_name)
831 return (test_name[0:index], test_name[index:])
833 def normalize_test_name(self, test_name):
834 """Returns a normalized version of the test name or test directory."""
835 if test_name.endswith('/'):
837 if self.test_isdir(test_name):
838 return test_name + '/'
841 def driver_cmd_line(self):
842 """Prints the DRT command line that will be used."""
843 driver = self.create_driver(0)
844 return driver.cmd_line(self.get_option('pixel_tests'), [])
846 def update_baseline(self, baseline_path, data):
847 """Updates the baseline for a test.
850 baseline_path: the actual path to use for baseline, not the path to
851 the test. This function is used to update either generic or
852 platform-specific baselines, but we can't infer which here.
853 data: contents of the baseline.
855 self._filesystem.write_binary_file(baseline_path, data)
857 # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected).
858 def webkit_base(self):
859 return self._webkit_finder.webkit_base()
861 def path_from_webkit_base(self, *comps):
862 return self._webkit_finder.path_from_webkit_base(*comps)
864 def path_from_chromium_base(self, *comps):
865 return self._webkit_finder.path_from_chromium_base(*comps)
867 def path_to_script(self, script_name):
868 return self._webkit_finder.path_to_script(script_name)
870 def layout_tests_dir(self):
871 return self._webkit_finder.layout_tests_dir()
873 def perf_tests_dir(self):
874 return self._webkit_finder.perf_tests_dir()
876 def skipped_layout_tests(self, test_list):
877 """Returns tests skipped outside of the TestExpectations files."""
878 return set(self._skipped_tests_for_unsupported_features(test_list))
880 def _tests_from_skipped_file_contents(self, skipped_file_contents):
882 for line in skipped_file_contents.split('\n'):
884 line = line.rstrip('/') # Best to normalize directory names to not include the trailing slash.
885 if line.startswith('#') or not len(line):
887 tests_to_skip.append(line)
890 def _expectations_from_skipped_files(self, skipped_file_paths):
892 for search_path in skipped_file_paths:
893 filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped")
894 if not self._filesystem.exists(filename):
895 _log.debug("Skipped does not exist: %s" % filename)
897 _log.debug("Using Skipped file: %s" % filename)
898 skipped_file_contents = self._filesystem.read_text_file(filename)
899 tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
903 def skipped_perf_tests(self):
904 return self._expectations_from_skipped_files([self.perf_tests_dir()])
906 def skips_perf_test(self, test_name):
907 for test_or_category in self.skipped_perf_tests():
908 if test_or_category == test_name:
910 category = self._filesystem.join(self.perf_tests_dir(), test_or_category)
911 if self._filesystem.isdir(category) and test_name.startswith(test_or_category):
915 def is_chromium(self):
919 """Returns a name that uniquely identifies this particular type of port
920 (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed
921 to factory.get() to instantiate the port."""
924 def operating_system(self):
925 # Subclasses should override this default implementation.
929 """Returns a string indicating the version of a given platform, e.g.
932 This is used to help identify the exact port when parsing test
933 expectations, determining search paths, and logging information."""
936 def architecture(self):
937 return self._architecture
939 def get_option(self, name, default_value=None):
940 return getattr(self._options, name, default_value)
942 def set_option_default(self, name, default_value):
943 return self._options.ensure_value(name, default_value)
946 def path_to_generic_test_expectations_file(self):
947 return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations')
949 def relative_test_filename(self, filename):
950 """Returns a test_name a relative unix-style path for a filename under the LayoutTests
951 directory. Ports may legitimately return abspaths here if no relpath makes sense."""
952 # Ports that run on windows need to override this method to deal with
953 # filenames with backslashes in them.
954 if filename.startswith(self.layout_tests_dir()):
955 return self.host.filesystem.relpath(filename, self.layout_tests_dir())
957 return self.host.filesystem.abspath(filename)
960 def abspath_for_test(self, test_name):
961 """Returns the full path to the file for a given test name. This is the
962 inverse of relative_test_filename()."""
963 return self._filesystem.join(self.layout_tests_dir(), test_name)
965 def results_directory(self):
966 """Absolute path to the place to store the test results (uses --results-directory)."""
967 if not self._results_directory:
968 option_val = self.get_option('results_directory') or self.default_results_directory()
969 self._results_directory = self._filesystem.abspath(option_val)
970 return self._results_directory
972 def perf_results_directory(self):
973 return self._build_path()
975 def default_results_directory(self):
976 """Absolute path to the default place to store the test results."""
978 return self.path_from_chromium_base('webkit', self.get_option('configuration'), 'layout-test-results')
979 except AssertionError:
980 return self._build_path('layout-test-results')
982 def setup_test_run(self):
983 """Perform port-specific work at the beginning of a test run."""
984 # Delete the disk cache if any to ensure a clean test run.
985 dump_render_tree_binary_path = self._path_to_driver()
986 cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
987 cachedir = self._filesystem.join(cachedir, "cache")
988 if self._filesystem.exists(cachedir):
989 self._filesystem.rmtree(cachedir)
991 if self._dump_reader:
992 self._filesystem.maybe_make_directory(self._dump_reader.crash_dumps_directory())
994 def num_workers(self, requested_num_workers):
995 """Returns the number of available workers (possibly less than the number requested)."""
996 return requested_num_workers
998 def clean_up_test_run(self):
999 """Perform port-specific work at the end of a test run."""
1000 if self._image_differ:
1001 self._image_differ.stop()
1002 self._image_differ = None
1004 # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable.
1005 def _value_or_default_from_environ(self, name, default=None):
1006 if name in os.environ:
1007 return os.environ[name]
1010 def _copy_value_from_environ_if_set(self, clean_env, name):
1011 if name in os.environ:
1012 clean_env[name] = os.environ[name]
1014 def setup_environ_for_server(self, server_name=None):
1015 # We intentionally copy only a subset of os.environ when
1016 # launching subprocesses to ensure consistent test results.
1018 'LOCAL_RESOURCE_ROOT': self.layout_tests_dir(), # FIXME: Is this used?
1020 variables_to_copy = [
1021 'WEBKIT_TESTFONTS', # FIXME: Is this still used?
1022 'WEBKITOUTPUTDIR', # FIXME: Is this still used?
1023 'CHROME_DEVEL_SANDBOX',
1024 'CHROME_IPC_LOGGING',
1027 'VALGRIND_LIB_INNER',
1029 if self.host.platform.is_linux() or self.host.platform.is_freebsd():
1030 variables_to_copy += [
1035 'DBUS_SESSION_BUS_ADDRESS',
1038 clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1')
1039 if self.host.platform.is_mac():
1040 clean_env['DYLD_LIBRARY_PATH'] = self._build_path()
1041 clean_env['DYLD_FRAMEWORK_PATH'] = self._build_path()
1042 variables_to_copy += [
1045 if self.host.platform.is_win():
1046 variables_to_copy += [
1048 'GYP_DEFINES', # Required to locate win sdk.
1050 if self.host.platform.is_cygwin():
1051 variables_to_copy += [
1057 for variable in variables_to_copy:
1058 self._copy_value_from_environ_if_set(clean_env, variable)
1060 for string_variable in self.get_option('additional_env_var', []):
1061 [name, value] = string_variable.split('=', 1)
1062 clean_env[name] = value
1066 def show_results_html_file(self, results_filename):
1067 """This routine should display the HTML file pointed at by
1068 results_filename in a users' browser."""
1069 return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename))
1071 def create_driver(self, worker_number, no_timeout=False):
1072 """Return a newly created Driver subclass for starting/stopping the test driver."""
1073 return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout)
1075 def start_helper(self):
1076 """If a port needs to reconfigure graphics settings or do other
1077 things to ensure a known test configuration, it should override this
1079 helper_path = self._path_to_helper()
1081 _log.debug("Starting layout helper %s" % helper_path)
1082 # Note: Not thread safe: http://bugs.python.org/issue2320
1083 self._helper = self._executive.popen([helper_path],
1084 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
1085 is_ready = self._helper.stdout.readline()
1086 if not is_ready.startswith('ready'):
1087 _log.error("layout_test_helper failed to be ready")
1089 def requires_http_server(self):
1090 """Does the port require an HTTP server for running tests? This could
1091 be the case when the tests aren't run on the host platform."""
1094 def start_http_server(self, additional_dirs=None, number_of_servers=None):
1095 """Start a web server. Raise an error if it can't start or is already running.
1097 Ports can stub this out if they don't need a web server to be running."""
1098 assert not self._http_server, 'Already running an http server.'
1100 if self._uses_apache():
1101 server = apache_http_server.LayoutTestApacheHttpd(self, self.results_directory(), additional_dirs=additional_dirs, number_of_servers=number_of_servers)
1103 server = http_server.Lighttpd(self, self.results_directory(), additional_dirs=additional_dirs, number_of_servers=number_of_servers)
1106 self._http_server = server
1108 def start_websocket_server(self):
1109 """Start a web server. Raise an error if it can't start or is already running.
1111 Ports can stub this out if they don't need a websocket server to be running."""
1112 assert not self._websocket_server, 'Already running a websocket server.'
1114 server = websocket_server.PyWebSocket(self, self.results_directory())
1116 self._websocket_server = server
1118 def http_server_supports_ipv6(self):
1119 # Cygwin is the only platform to still use Apache 1.3, which only supports IPV4.
1120 # Once it moves to Apache 2, we can drop this method altogether.
1121 if self.host.platform.is_cygwin():
1125 def stop_helper(self):
1126 """Shut down the test helper if it is running. Do nothing if
1127 it isn't, or it isn't available. If a port overrides start_helper()
1128 it must override this routine as well."""
1130 _log.debug("Stopping layout test helper")
1132 self._helper.stdin.write("x\n")
1133 self._helper.stdin.close()
1140 def stop_http_server(self):
1141 """Shut down the http server if it is running. Do nothing if it isn't."""
1142 if self._http_server:
1143 self._http_server.stop()
1144 self._http_server = None
1146 def stop_websocket_server(self):
1147 """Shut down the websocket server if it is running. Do nothing if it isn't."""
1148 if self._websocket_server:
1149 self._websocket_server.stop()
1150 self._websocket_server = None
1153 # TEST EXPECTATION-RELATED METHODS
1156 def test_configuration(self):
1157 """Returns the current TestConfiguration for the port."""
1158 if not self._test_configuration:
1159 self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower())
1160 return self._test_configuration
1162 # FIXME: Belongs on a Platform object.
1164 def all_test_configurations(self):
1165 """Returns a list of TestConfiguration instances, representing all available
1166 test configurations for this port."""
1167 return self._generate_all_test_configurations()
1169 # FIXME: Belongs on a Platform object.
1170 def configuration_specifier_macros(self):
1171 """Ports may provide a way to abbreviate configuration specifiers to conveniently
1172 refer to them as one term or alias specific values to more generic ones. For example:
1174 (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake.
1175 (lucid) -> linux # Change specific name of the Linux distro to a more generic term.
1177 Returns a dictionary, each key representing a macro term ('win', for example),
1178 and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7'])."""
1179 return self.CONFIGURATION_SPECIFIER_MACROS
1181 def all_baseline_variants(self):
1182 """Returns a list of platform names sufficient to cover all the baselines.
1184 The list should be sorted so that a later platform will reuse
1185 an earlier platform's baselines if they are the same (e.g.,
1186 'snowleopard' should precede 'leopard')."""
1187 return self.ALL_BASELINE_VARIANTS
1189 def _generate_all_test_configurations(self):
1190 """Returns a sequence of the TestConfigurations the port supports."""
1191 # By default, we assume we want to test every graphics type in
1192 # every configuration on every system.
1193 test_configurations = []
1194 for version, architecture in self.ALL_SYSTEMS:
1195 for build_type in self.ALL_BUILD_TYPES:
1196 test_configurations.append(TestConfiguration(version, architecture, build_type))
1197 return test_configurations
1199 try_builder_names = frozenset([
1208 def warn_if_bug_missing_in_test_expectations(self):
1211 def _port_specific_expectations_files(self):
1213 paths.append(self.path_from_chromium_base('skia', 'skia_test_expectations.txt'))
1214 paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations_w3c.txt'))
1215 paths.append(self._filesystem.join(self.layout_tests_dir(), 'NeverFixTests'))
1216 paths.append(self._filesystem.join(self.layout_tests_dir(), 'StaleTestExpectations'))
1217 paths.append(self._filesystem.join(self.layout_tests_dir(), 'SlowTests'))
1219 builder_name = self.get_option('builder_name', 'DUMMY_BUILDER_NAME')
1220 if builder_name == 'DUMMY_BUILDER_NAME' or '(deps)' in builder_name or builder_name in self.try_builder_names:
1221 paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations.txt'))
1224 def expectations_dict(self):
1225 """Returns an OrderedDict of name -> expectations strings.
1226 The names are expected to be (but not required to be) paths in the filesystem.
1227 If the name is a path, the file can be considered updatable for things like rebaselining,
1228 so don't use names that are paths if they're not paths.
1229 Generally speaking the ordering should be files in the filesystem in cascade order
1230 (TestExpectations followed by Skipped, if the port honors both formats),
1231 then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options."""
1232 # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict.
1233 expectations = OrderedDict()
1235 for path in self.expectations_files():
1236 if self._filesystem.exists(path):
1237 expectations[path] = self._filesystem.read_text_file(path)
1239 for path in self.get_option('additional_expectations', []):
1240 expanded_path = self._filesystem.expanduser(path)
1241 if self._filesystem.exists(expanded_path):
1242 _log.debug("reading additional_expectations from path '%s'" % path)
1243 expectations[path] = self._filesystem.read_text_file(expanded_path)
1245 _log.warning("additional_expectations path '%s' does not exist" % path)
1248 def bot_expectations(self):
1249 if not self.get_option('ignore_flaky_tests'):
1252 full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name)
1253 builder_category = self.get_option('ignore_builder_category', 'layout')
1254 factory = BotTestExpectationsFactory()
1255 expectations = factory.expectations_for_port(full_port_name, builder_category)
1257 if not expectations:
1260 ignore_mode = self.get_option('ignore_flaky_tests')
1261 if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky':
1262 return expectations.flakes_by_path(ignore_mode == 'very-flaky')
1263 if ignore_mode == 'unexpected':
1264 return expectations.unexpected_results_by_path()
1265 _log.warning("Unexpected ignore mode: '%s'." % ignore_mode)
1268 def expectations_files(self):
1269 return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files()
1271 def repository_paths(self):
1272 """Returns a list of (repository_name, repository_path) tuples of its depending code base."""
1273 return [('blink', self.layout_tests_dir()),
1274 ('chromium', self.path_from_chromium_base('build'))]
1276 _WDIFF_DEL = '##WDIFF_DEL##'
1277 _WDIFF_ADD = '##WDIFF_ADD##'
1278 _WDIFF_END = '##WDIFF_END##'
1280 def _format_wdiff_output_as_html(self, wdiff):
1281 wdiff = cgi.escape(wdiff)
1282 wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
1283 wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
1284 wdiff = wdiff.replace(self._WDIFF_END, "</span>")
1285 html = "<head><style>.del { background: #faa; } "
1286 html += ".add { background: #afa; }</style></head>"
1287 html += "<pre>%s</pre>" % wdiff
1290 def _wdiff_command(self, actual_filename, expected_filename):
1291 executable = self._path_to_wdiff()
1293 "--start-delete=%s" % self._WDIFF_DEL,
1294 "--end-delete=%s" % self._WDIFF_END,
1295 "--start-insert=%s" % self._WDIFF_ADD,
1296 "--end-insert=%s" % self._WDIFF_END,
1301 def _handle_wdiff_error(script_error):
1302 # Exit 1 means the files differed, any other exit code is an error.
1303 if script_error.exit_code != 1:
1306 def _run_wdiff(self, actual_filename, expected_filename):
1307 """Runs wdiff and may throw exceptions.
1308 This is mostly a hook for unit testing."""
1309 # Diffs are treated as binary as they may include multiple files
1310 # with conflicting encodings. Thus we do not decode the output.
1311 command = self._wdiff_command(actual_filename, expected_filename)
1312 wdiff = self._executive.run_command(command, decode_output=False,
1313 error_handler=self._handle_wdiff_error)
1314 return self._format_wdiff_output_as_html(wdiff)
1316 _wdiff_error_html = "Failed to run wdiff, see error log."
1318 def wdiff_text(self, actual_filename, expected_filename):
1319 """Returns a string of HTML indicating the word-level diff of the
1320 contents of the two filenames. Returns an empty string if word-level
1321 diffing isn't available."""
1322 if not self.wdiff_available():
1325 # It's possible to raise a ScriptError we pass wdiff invalid paths.
1326 return self._run_wdiff(actual_filename, expected_filename)
1327 except OSError as e:
1328 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
1329 # Silently ignore cases where wdiff is missing.
1330 self._wdiff_available = False
1333 except ScriptError as e:
1334 _log.error("Failed to run wdiff: %s" % e)
1335 self._wdiff_available = False
1336 return self._wdiff_error_html
1338 # This is a class variable so we can test error output easily.
1339 _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
1341 def pretty_patch_text(self, diff_path):
1342 if self._pretty_patch_available is None:
1343 self._pretty_patch_available = self.check_pretty_patch(logging=False)
1344 if not self._pretty_patch_available:
1345 return self._pretty_patch_error_html
1346 command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path),
1347 self._pretty_patch_path, diff_path)
1349 # Diffs are treated as binary (we pass decode_output=False) as they
1350 # may contain multiple files of conflicting encodings.
1351 return self._executive.run_command(command, decode_output=False)
1353 # If the system is missing ruby log the error and stop trying.
1354 self._pretty_patch_available = False
1355 _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
1356 return self._pretty_patch_error_html
1357 except ScriptError, e:
1358 # If ruby failed to run for some reason, log the command
1359 # output and stop trying.
1360 self._pretty_patch_available = False
1361 _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output()))
1362 return self._pretty_patch_error_html
1364 def default_configuration(self):
1365 return self._config.default_configuration()
1367 def clobber_old_port_specific_results(self):
1371 # PROTECTED ROUTINES
1373 # The routines below should only be called by routines in this class
1374 # or any of its subclasses.
1377 def _uses_apache(self):
1380 # FIXME: This does not belong on the port object.
1382 def _path_to_apache(self):
1383 """Returns the full path to the apache binary.
1385 This is needed only by ports that use the apache_http_server module."""
1386 raise NotImplementedError('Port._path_to_apache')
1388 # FIXME: This belongs on some platform abstraction instead of Port.
1389 def _is_redhat_based(self):
1390 return self._filesystem.exists('/etc/redhat-release')
1392 def _is_debian_based(self):
1393 return self._filesystem.exists('/etc/debian_version')
1395 def _apache_version(self):
1396 config = self._executive.run_command([self._path_to_apache(), '-v'])
1397 return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config)
1399 # We pass sys_platform into this method to make it easy to unit test.
1400 def _apache_config_file_name_for_platform(self, sys_platform):
1401 if sys_platform == 'cygwin':
1402 return 'cygwin-httpd.conf' # CYGWIN is the only platform to still use Apache 1.3.
1403 if sys_platform.startswith('linux'):
1404 if self._is_redhat_based():
1405 return 'fedora-httpd-' + self._apache_version() + '.conf'
1406 if self._is_debian_based():
1407 return 'debian-httpd-' + self._apache_version() + '.conf'
1408 # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
1409 return "apache2-httpd.conf"
1411 def _path_to_apache_config_file(self):
1412 """Returns the full path to the apache configuration file.
1414 If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its
1415 contents will be used instead.
1417 This is needed only by ports that use the apache_http_server module."""
1418 config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH')
1419 if config_file_from_env:
1420 if not self._filesystem.exists(config_file_from_env):
1421 raise IOError('%s was not found on the system' % config_file_from_env)
1422 return config_file_from_env
1424 config_file_name = self._apache_config_file_name_for_platform(sys.platform)
1425 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
1427 def _path_to_driver(self, configuration=None):
1428 """Returns the full path to the test driver."""
1429 return self._build_path(self.driver_name())
1431 def _path_to_webcore_library(self):
1432 """Returns the full path to a built copy of WebCore."""
1435 def _path_to_helper(self):
1436 """Returns the full path to the layout_test_helper binary, which
1437 is used to help configure the system for the test run, or None
1438 if no helper is needed.
1440 This is likely only used by start/stop_helper()."""
1443 def _path_to_image_diff(self):
1444 """Returns the full path to the image_diff binary, or None if it is not available.
1446 This is likely used only by diff_image()"""
1447 return self._build_path('image_diff')
1449 def _path_to_lighttpd(self):
1450 """Returns the path to the LigHTTPd binary.
1452 This is needed only by ports that use the http_server.py module."""
1453 raise NotImplementedError('Port._path_to_lighttpd')
1455 def _path_to_lighttpd_modules(self):
1456 """Returns the path to the LigHTTPd modules directory.
1458 This is needed only by ports that use the http_server.py module."""
1459 raise NotImplementedError('Port._path_to_lighttpd_modules')
1461 def _path_to_lighttpd_php(self):
1462 """Returns the path to the LigHTTPd PHP executable.
1464 This is needed only by ports that use the http_server.py module."""
1465 raise NotImplementedError('Port._path_to_lighttpd_php')
1468 def _path_to_wdiff(self):
1469 """Returns the full path to the wdiff binary, or None if it is not available.
1471 This is likely used only by wdiff_text()"""
1472 for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"):
1473 if self._filesystem.exists(path):
1477 def _webkit_baseline_path(self, platform):
1478 """Return the full path to the top of the baseline tree for a
1480 return self._filesystem.join(self.layout_tests_dir(), 'platform', platform)
1482 def _driver_class(self):
1483 """Returns the port's driver implementation."""
1484 return driver.Driver
1486 def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
1487 if stderr and 'AddressSanitizer' in stderr:
1488 # Running the AddressSanitizer take a lot of memory, so we need to
1489 # serialize access to it across all the concurrently running drivers.
1491 # FIXME: investigate using LLVM_SYMBOLIZER_PATH here to reduce the overhead.
1492 asan_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
1493 if self._filesystem.exists(asan_filter_path):
1494 output = self._executive.run_command(['flock', sys.executable, asan_filter_path], input=stderr, decode_output=False)
1495 stderr = self._executive.run_command(['c++filt'], input=output, decode_output=False)
1497 name_str = name or '<unknown process name>'
1498 pid_str = str(pid or '<unknown>')
1499 stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines()
1500 stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines()
1501 return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str,
1502 '\n'.join(('STDOUT: ' + l) for l in stdout_lines),
1503 '\n'.join(('STDERR: ' + l) for l in stderr_lines)))
1505 def look_for_new_crash_logs(self, crashed_processes, start_time):
1508 def look_for_new_samples(self, unresponsive_processes, start_time):
1511 def sample_process(self, name, pid):
1514 def virtual_test_suites(self):
1516 VirtualTestSuite('gpu',
1518 ['--enable-accelerated-2d-canvas']),
1519 VirtualTestSuite('gpu',
1521 ['--enable-accelerated-2d-canvas']),
1522 VirtualTestSuite('threaded',
1523 'compositing/visibility',
1524 ['--enable-threaded-compositing']),
1525 VirtualTestSuite('threaded',
1526 'compositing/webgl',
1527 ['--enable-threaded-compositing']),
1528 VirtualTestSuite('gpu',
1530 ['--force-compositing-mode']),
1531 VirtualTestSuite('softwarecompositing',
1533 ['--enable-software-compositing', '--disable-gpu-compositing'],
1534 use_legacy_naming=True),
1535 VirtualTestSuite('deferred',
1537 ['--enable-deferred-image-decoding', '--enable-per-tile-painting', '--force-compositing-mode']),
1538 VirtualTestSuite('deferred',
1539 'inspector/timeline',
1540 ['--enable-deferred-image-decoding', '--enable-per-tile-painting', '--force-compositing-mode']),
1541 VirtualTestSuite('gpu/compositedscrolling/overflow',
1542 'compositing/overflow',
1543 ['--enable-accelerated-overflow-scroll'],
1544 use_legacy_naming=True),
1545 VirtualTestSuite('gpu/compositedscrolling/scrollbars',
1547 ['--enable-accelerated-overflow-scroll'],
1548 use_legacy_naming=True),
1549 VirtualTestSuite('threaded',
1551 ['--enable-threaded-compositing']),
1552 VirtualTestSuite('threaded',
1554 ['--enable-threaded-compositing']),
1555 VirtualTestSuite('legacy-animations-engine',
1557 ['--disable-web-animations-css']),
1558 VirtualTestSuite('legacy-animations-engine',
1560 ['--disable-web-animations-css']),
1561 VirtualTestSuite('stable',
1563 ['--stable-release-mode']),
1564 VirtualTestSuite('stable',
1566 ['--stable-release-mode']),
1567 VirtualTestSuite('android',
1569 ['--force-compositing-mode', '--allow-webui-compositing', '--enable-threaded-compositing',
1570 '--enable-fixed-position-compositing', '--enable-accelerated-overflow-scroll', '--enable-accelerated-scrollable-frames',
1571 '--enable-composited-scrolling-for-frames', '--enable-gesture-tap-highlight', '--enable-pinch',
1572 '--enable-overlay-fullscreen-video', '--enable-overlay-scrollbars', '--enable-overscroll-notifications',
1573 '--enable-fixed-layout', '--enable-viewport', '--disable-canvas-aa',
1574 '--disable-composited-antialiasing']),
1575 VirtualTestSuite('implsidepainting',
1576 'inspector/timeline',
1577 ['--enable-threaded-compositing', '--enable-impl-side-painting', '--force-compositing-mode']),
1578 VirtualTestSuite('fasttextautosizing',
1579 'fast/text-autosizing',
1580 ['--enable-fast-text-autosizing']),
1581 VirtualTestSuite('serviceworker',
1582 'http/tests/serviceworker',
1583 ['--enable-service-worker']),
1587 def populated_virtual_test_suites(self):
1588 suites = self.virtual_test_suites()
1590 # Sanity-check the suites to make sure they don't point to other suites.
1591 suite_dirs = [suite.name for suite in suites]
1592 for suite in suites:
1593 assert suite.base not in suite_dirs
1595 for suite in suites:
1596 base_tests = self._real_tests([suite.base])
1598 for test in base_tests:
1599 suite.tests[test.replace(suite.base, suite.name, 1)] = test
1602 def _virtual_tests(self, paths, suites):
1603 virtual_tests = list()
1604 for suite in suites:
1606 for test in suite.tests:
1607 if any(test.startswith(p) for p in paths):
1608 virtual_tests.append(test)
1610 virtual_tests.extend(suite.tests.keys())
1611 return virtual_tests
1613 def is_virtual_test(self, test_name):
1614 return bool(self.lookup_virtual_suite(test_name))
1616 def lookup_virtual_suite(self, test_name):
1617 for suite in self.populated_virtual_test_suites():
1618 if test_name.startswith(suite.name):
1622 def lookup_virtual_test_base(self, test_name):
1623 suite = self.lookup_virtual_suite(test_name)
1626 return test_name.replace(suite.name, suite.base, 1)
1628 def lookup_virtual_test_args(self, test_name):
1629 for suite in self.populated_virtual_test_suites():
1630 if test_name.startswith(suite.name):
1634 def should_run_as_pixel_test(self, test_input):
1635 if not self._options.pixel_tests:
1637 if self._options.pixel_test_directories:
1638 return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories)
1641 def _modules_to_search_for_symbols(self):
1642 path = self._path_to_webcore_library()
1647 def _symbols_string(self):
1649 for path_to_module in self._modules_to_search_for_symbols():
1651 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error)
1653 _log.warn("Failed to run nm: %s. Can't determine supported features correctly." % e)
1656 # Ports which use compile-time feature detection should define this method and return
1657 # a dictionary mapping from symbol substrings to possibly disabled test directories.
1658 # When the symbol substrings are not matched, the directories will be skipped.
1659 # If ports don't ever enable certain features, then those directories can just be
1660 # in the Skipped list instead of compile-time-checked here.
1661 def _missing_symbol_to_skipped_tests(self):
1662 if self.PORT_HAS_AUDIO_CODECS_BUILT_IN:
1666 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"],
1667 "ff_aac_decoder": ["webaudio/codec-tests/aac"],
1670 def _has_test_in_directories(self, directory_lists, test_list):
1674 directories = itertools.chain.from_iterable(directory_lists)
1675 for directory, test in itertools.product(directories, test_list):
1676 if test.startswith(directory):
1680 def _skipped_tests_for_unsupported_features(self, test_list):
1681 # Only check the symbols of there are tests in the test_list that might get skipped.
1682 # This is a performance optimization to avoid the calling nm.
1683 # Runtime feature detection not supported, fallback to static detection:
1684 # Disable any tests for symbols missing from the executable or libraries.
1685 if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
1686 symbols_string = self._symbols_string()
1687 if symbols_string is not None:
1688 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], [])
1691 def _convert_path(self, path):
1692 """Handles filename conversion for subprocess command line args."""
1693 # See note above in diff_image() for why we need this.
1694 if sys.platform == 'cygwin':
1695 return cygpath(path)
1698 def _build_path(self, *comps):
1699 return self._build_path_with_configuration(None, *comps)
1701 def _build_path_with_configuration(self, configuration, *comps):
1702 # Note that we don't do the option caching that the
1703 # base class does, because finding the right directory is relatively
1705 configuration = configuration or self.get_option('configuration')
1706 return self._static_build_path(self._filesystem, self.get_option('build_directory'),
1707 self.path_from_chromium_base(), configuration, comps)
1709 def _check_driver_build_up_to_date(self, configuration):
1710 if configuration in ('Debug', 'Release'):
1712 debug_path = self._path_to_driver('Debug')
1713 release_path = self._path_to_driver('Release')
1715 debug_mtime = self._filesystem.mtime(debug_path)
1716 release_mtime = self._filesystem.mtime(release_path)
1718 if (debug_mtime > release_mtime and configuration == 'Release' or
1719 release_mtime > debug_mtime and configuration == 'Debug'):
1720 most_recent_binary = 'Release' if configuration == 'Debug' else 'Debug'
1721 _log.warning('You are running the %s binary. However the %s binary appears to be more recent. '
1722 'Please pass --%s.', configuration, most_recent_binary, most_recent_binary.lower())
1724 # This will fail if we don't have both a debug and release binary.
1725 # That's fine because, in this case, we must already be running the
1726 # most up-to-date one.
1731 def _chromium_baseline_path(self, platform):
1732 if platform is None:
1733 platform = self.name()
1734 return self.path_from_webkit_base('LayoutTests', 'platform', platform)
1736 class VirtualTestSuite(object):
1737 def __init__(self, name, base, args, use_legacy_naming=False, tests=None):
1738 if use_legacy_naming:
1739 self.name = 'virtual/' + name
1741 if name.find('/') != -1:
1742 _log.error("Virtual test suites names cannot contain /'s: %s" % name)
1744 self.name = 'virtual/' + name + '/' + base
1747 self.tests = tests or set()
1750 return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)