1 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
7 # * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 # * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
13 # * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 """Abstract base class of Port-specific entry points for the layout tests
30 test infrastructure (the Port and Driver classes)."""
45 from collections import OrderedDict
47 # Needed for Python < 2.7
48 from webkitpy.thirdparty.ordered_dict import OrderedDict
51 from webkitpy.common import find_files
52 from webkitpy.common import read_checksum_from_png
53 from webkitpy.common.memoized import memoized
54 from webkitpy.common.system import path
55 from webkitpy.common.system.executive import ScriptError
56 from webkitpy.common.system.path import cygpath
57 from webkitpy.common.system.systemhost import SystemHost
58 from webkitpy.common.webkit_finder import WebKitFinder
59 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
60 from webkitpy.layout_tests.models import test_run_results
61 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
62 from webkitpy.layout_tests.port import config as port_config
63 from webkitpy.layout_tests.port import driver
64 from webkitpy.layout_tests.port import server_process
65 from webkitpy.layout_tests.port.factory import PortFactory
66 from webkitpy.layout_tests.servers import apache_http
67 from webkitpy.layout_tests.servers import pywebsocket
69 _log = logging.getLogger(__name__)
72 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports.
74 """Abstract class for Port-specific hooks for the layout_test package."""
76 # Subclasses override this. This should indicate the basic implementation
77 # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?)
78 # one unique value per class.
80 # FIXME: We should probably rename this to something like 'implementation_name'.
83 # Test names resemble unix relative paths, and use '/' as a directory separator.
84 TEST_PATH_SEPARATOR = '/'
86 ALL_BUILD_TYPES = ('debug', 'release')
88 CONTENT_SHELL_NAME = 'content_shell'
90 # True if the port as aac and mp3 codecs built in.
91 PORT_HAS_AUDIO_CODECS_BUILT_IN = False
94 ('snowleopard', 'x86'),
97 # FIXME: We treat Retina (High-DPI) devices as if they are running
98 # a different operating system version. This isn't accurate, but will work until
99 # we need to test and support baselines across multiple O/S versions.
102 ('mountainlion', 'x86'),
103 ('mavericks', 'x86'),
108 # FIXME: Technically this should be 'arm', but adding a third architecture type breaks TestConfigurationConverter.
109 # If we need this to be 'arm' in the future, then we first have to fix TestConfigurationConverter.
110 ('icecreamsandwich', 'x86'),
113 ALL_BASELINE_VARIANTS = [
114 'mac-mavericks', 'mac-mountainlion', 'mac-retina', 'mac-lion', 'mac-snowleopard',
115 'win-win7', 'win-xp',
116 'linux-x86_64', 'linux-x86',
119 CONFIGURATION_SPECIFIER_MACROS = {
120 'mac': ['snowleopard', 'lion', 'retina', 'mountainlion', 'mavericks'],
121 'win': ['xp', 'win7'],
123 'android': ['icecreamsandwich'],
126 DEFAULT_BUILD_DIRECTORIES = ('out',)
128 # overridden in subclasses.
131 SUPPORTED_VERSIONS = []
133 # URL to the build requirements page.
134 BUILD_REQUIREMENTS_URL = ''
137 def latest_platform_fallback_path(cls):
138 return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]]
141 def _static_build_path(cls, filesystem, build_directory, chromium_base, configuration, comps):
143 return filesystem.join(build_directory, configuration, *comps)
146 for directory in cls.DEFAULT_BUILD_DIRECTORIES:
147 base_dir = filesystem.join(chromium_base, directory, configuration)
148 path = filesystem.join(base_dir, *comps)
149 if filesystem.exists(path):
150 hits.append((filesystem.mtime(path), path))
153 hits.sort(reverse=True)
154 return hits[0][1] # Return the newest file found.
156 # We have to default to something, so pick the last one.
157 return filesystem.join(base_dir, *comps)
160 def determine_full_port_name(cls, host, options, port_name):
161 """Return a fully-specified port name that can be used to construct objects."""
162 # Subclasses will usually override this.
163 assert port_name.startswith(cls.port_name)
166 def __init__(self, host, port_name, options=None, **kwargs):
168 # This value may be different from cls.port_name by having version modifiers
169 # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2').
170 self._name = port_name
172 # These are default values that should be overridden in a subclasses.
174 self._architecture = 'x86'
176 # FIXME: Ideally we'd have a package-wide way to get a
177 # well-formed options object that had all of the necessary
178 # options defined on it.
179 self._options = options or optparse.Values()
182 self._executive = host.executive
183 self._filesystem = host.filesystem
184 self._webkit_finder = WebKitFinder(host.filesystem)
185 self._config = port_config.Config(self._executive, self._filesystem, self.port_name)
188 self._http_server = None
189 self._websocket_server = None
190 self._image_differ = None
191 self._server_process_constructor = server_process.ServerProcess # overridable for testing
192 self._http_lock = None # FIXME: Why does this live on the port object?
193 self._dump_reader = None
195 # Python's Popen has a bug that causes any pipes opened to a
196 # process that can't be executed to be leaked. Since this
197 # code is specifically designed to tolerate exec failures
198 # to gracefully handle cases where wdiff is not installed,
199 # the bug results in a massive file descriptor leak. As a
200 # workaround, if an exec failure is ever experienced for
201 # wdiff, assume it's not available. This will leak one
202 # file descriptor but that's better than leaking each time
203 # wdiff would be run.
205 # http://mail.python.org/pipermail/python-list/
206 # 2008-August/505753.html
207 # http://bugs.python.org/issue3210
208 self._wdiff_available = None
210 # FIXME: prettypatch.py knows this path, why is it copied here?
211 self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb")
212 self._pretty_patch_available = None
214 if not hasattr(options, 'configuration') or not options.configuration:
215 self.set_option_default('configuration', self.default_configuration())
216 self._test_configuration = None
217 self._reftest_list = {}
218 self._results_directory = None
219 self._virtual_test_suites = None
221 def buildbot_archives_baselines(self):
224 def additional_drt_flag(self):
225 if self.driver_name() == self.CONTENT_SHELL_NAME:
226 return ['--dump-render-tree']
229 def supports_per_test_timeout(self):
232 def default_pixel_tests(self):
235 def default_smoke_test_only(self):
238 def default_timeout_ms(self):
239 timeout_ms = 6 * 1000
240 if self.get_option('configuration') == 'Debug':
241 # Debug is usually 2x-3x slower than Release.
242 return 3 * timeout_ms
245 def driver_stop_timeout(self):
246 """ Returns the amount of time in seconds to wait before killing the process in driver.stop()."""
247 # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as
248 # well (for things like ASAN, Valgrind, etc.)
249 return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms()
251 def wdiff_available(self):
252 if self._wdiff_available is None:
253 self._wdiff_available = self.check_wdiff(logging=False)
254 return self._wdiff_available
256 def pretty_patch_available(self):
257 if self._pretty_patch_available is None:
258 self._pretty_patch_available = self.check_pretty_patch(logging=False)
259 return self._pretty_patch_available
261 def default_child_processes(self):
262 """Return the number of drivers to use for this port."""
263 return self._executive.cpu_count()
265 def default_max_locked_shards(self):
266 """Return the number of "locked" shards to run in parallel (like the http tests)."""
267 max_locked_shards = int(self.default_child_processes()) / 4
268 if not max_locked_shards:
270 return max_locked_shards
272 def baseline_path(self):
273 """Return the absolute path to the directory to store new baselines in for this port."""
274 # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
275 return self.baseline_version_dir()
277 def baseline_platform_dir(self):
278 """Return the absolute path to the default (version-independent) platform-specific results."""
279 return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
281 def baseline_version_dir(self):
282 """Return the absolute path to the platform-and-version-specific results."""
283 baseline_search_paths = self.baseline_search_path()
284 return baseline_search_paths[0]
286 def virtual_baseline_search_path(self, test_name):
287 suite = self.lookup_virtual_suite(test_name)
290 return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()]
292 def baseline_search_path(self):
293 return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path()
295 def default_baseline_search_path(self):
296 """Return a list of absolute paths to directories to search under for
297 baselines. The directories are searched in order."""
298 return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version()])
301 def _compare_baseline(self):
302 factory = PortFactory(self.host)
303 target_port = self.get_option('compare_port')
305 return factory.get(target_port).default_baseline_search_path()
308 def _check_file_exists(self, path_to_file, file_description,
309 override_step=None, logging=True):
310 """Verify the file is present where expected or log an error.
313 file_name: The (human friendly) name or description of the file
314 you're looking for (e.g., "HTTP Server"). Used for error logging.
315 override_step: An optional string to be logged if the check fails.
316 logging: Whether or not log the error messages."""
317 if not self._filesystem.exists(path_to_file):
319 _log.error('Unable to find %s' % file_description)
320 _log.error(' at %s' % path_to_file)
322 _log.error(' %s' % override_step)
327 def check_build(self, needs_http, printer):
330 dump_render_tree_binary_path = self._path_to_driver()
331 result = self._check_file_exists(dump_render_tree_binary_path,
332 'test driver') and result
333 if not result and self.get_option('build'):
334 result = self._check_driver_build_up_to_date(
335 self.get_option('configuration'))
339 helper_path = self._path_to_helper()
341 result = self._check_file_exists(helper_path,
342 'layout test helper') and result
344 if self.get_option('pixel_tests'):
345 result = self.check_image_diff(
346 'To override, invoke with --no-pixel-tests') and result
348 # It's okay if pretty patch and wdiff aren't available, but we will at least log messages.
349 self._pretty_patch_available = self.check_pretty_patch()
350 self._wdiff_available = self.check_wdiff()
352 if self._dump_reader:
353 result = self._dump_reader.check_is_functional() and result
356 result = self.check_httpd() and result
358 return test_run_results.OK_EXIT_STATUS if result else test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
360 def _check_driver(self):
361 driver_path = self._path_to_driver()
362 if not self._filesystem.exists(driver_path):
363 _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
367 def _check_port_build(self):
368 # Ports can override this method to do additional checks.
371 def check_sys_deps(self, needs_http):
372 """If the port needs to do some runtime checks to ensure that the
373 tests can be run successfully, it should override this routine.
374 This step can be skipped with --nocheck-sys-deps.
376 Returns whether the system is properly configured."""
377 cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
379 local_error = ScriptError()
381 def error_handler(script_error):
382 local_error.exit_code = script_error.exit_code
384 output = self._executive.run_command(cmd, error_handler=error_handler)
385 if local_error.exit_code:
386 _log.error('System dependencies check failed.')
387 _log.error('To override, invoke with --nocheck-sys-deps')
390 if self.BUILD_REQUIREMENTS_URL is not '':
392 _log.error('For complete build requirements, please see:')
393 _log.error(self.BUILD_REQUIREMENTS_URL)
394 return test_run_results.SYS_DEPS_EXIT_STATUS
395 return test_run_results.OK_EXIT_STATUS
397 def check_image_diff(self, override_step=None, logging=True):
398 """This routine is used to check whether image_diff binary exists."""
399 image_diff_path = self._path_to_image_diff()
400 if not self._filesystem.exists(image_diff_path):
401 _log.error("image_diff was not found at %s" % image_diff_path)
405 def check_pretty_patch(self, logging=True):
406 """Checks whether we can use the PrettyPatch ruby script."""
408 _ = self._executive.run_command(['ruby', '--version'])
410 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
412 _log.warning("Ruby is not installed; can't generate pretty patches.")
416 if not self._filesystem.exists(self._pretty_patch_path):
418 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path)
424 def check_wdiff(self, logging=True):
425 if not self._path_to_wdiff():
426 # Don't need to log here since this is the port choosing not to use wdiff.
430 _ = self._executive.run_command([self._path_to_wdiff(), '--help'])
433 message = self._wdiff_missing_message()
435 for line in message.splitlines():
436 _log.warning(' ' + line)
442 def _wdiff_missing_message(self):
443 return 'wdiff is not installed; please install it to generate word-by-word diffs.'
445 def check_httpd(self):
446 httpd_path = self.path_to_apache()
448 server_name = self._filesystem.basename(httpd_path)
449 env = self.setup_environ_for_server(server_name)
450 if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0:
451 _log.error("httpd seems broken. Cannot run http tests.")
455 _log.error("No httpd found. Cannot run http tests.")
458 def do_text_results_differ(self, expected_text, actual_text):
459 return expected_text != actual_text
461 def do_audio_results_differ(self, expected_audio, actual_audio):
462 return expected_audio != actual_audio
464 def diff_image(self, expected_contents, actual_contents):
465 """Compare two images and return a tuple of an image diff, and an error string.
467 If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff).
469 # If only one of them exists, return that one.
470 if not actual_contents and not expected_contents:
472 if not actual_contents:
473 return (expected_contents, None)
474 if not expected_contents:
475 return (actual_contents, None)
477 tempdir = self._filesystem.mkdtemp()
479 expected_filename = self._filesystem.join(str(tempdir), "expected.png")
480 self._filesystem.write_binary_file(expected_filename, expected_contents)
482 actual_filename = self._filesystem.join(str(tempdir), "actual.png")
483 self._filesystem.write_binary_file(actual_filename, actual_contents)
485 diff_filename = self._filesystem.join(str(tempdir), "diff.png")
487 # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin.
488 native_expected_filename = self._convert_path(expected_filename)
489 native_actual_filename = self._convert_path(actual_filename)
490 native_diff_filename = self._convert_path(diff_filename)
492 executable = self._path_to_image_diff()
493 # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'.
494 comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename]
499 exit_code = self._executive.run_command(comand, return_exit_code=True)
501 # The images are the same.
504 result = self._filesystem.read_binary_file(native_diff_filename)
506 err_str = "Image diff returned an exit code of %s. See http://crbug.com/278596" % exit_code
508 err_str = 'error running image diff: %s' % str(e)
510 self._filesystem.rmtree(str(tempdir))
512 return (result, err_str or None)
514 def diff_text(self, expected_text, actual_text, expected_filename, actual_filename):
515 """Returns a string containing the diff of the two text strings
516 in 'unified diff' format."""
518 # The filenames show up in the diff output, make sure they're
519 # raw bytes and not unicode, so that they don't trigger join()
520 # trying to decode the input.
521 def to_raw_bytes(string_value):
522 if isinstance(string_value, unicode):
523 return string_value.encode('utf-8')
525 expected_filename = to_raw_bytes(expected_filename)
526 actual_filename = to_raw_bytes(actual_filename)
527 diff = difflib.unified_diff(expected_text.splitlines(True),
528 actual_text.splitlines(True),
532 # The diff generated by the difflib is incorrect if one of the files
533 # does not have a newline at the end of the file and it is present in
534 # the diff. Relevant Python issue: http://bugs.python.org/issue2142
535 def diff_fixup(diff):
538 if not line.endswith('\n'):
539 yield '\n\ No newline at end of file\n'
541 return ''.join(diff_fixup(diff))
543 def driver_name(self):
544 if self.get_option('driver_name'):
545 return self.get_option('driver_name')
546 return self.CONTENT_SHELL_NAME
548 def expected_baselines_by_extension(self, test_name):
549 """Returns a dict mapping baseline suffix to relative path for each baseline in
550 a test. For reftests, it returns ".==" or ".!=" instead of the suffix."""
551 # FIXME: The name similarity between this and expected_baselines() below, is unfortunate.
552 # We should probably rename them both.
554 reference_files = self.reference_files(test_name)
556 # FIXME: How should this handle more than one type of reftest?
557 baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1])
559 for extension in self.baseline_extensions():
560 path = self.expected_filename(test_name, extension, return_default=False)
561 baseline_dict[extension] = self.relative_test_filename(path) if path else path
565 def baseline_extensions(self):
566 """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'."""
567 return ('.wav', '.txt', '.png')
569 def expected_baselines(self, test_name, suffix, all_baselines=False):
570 """Given a test name, finds where the baseline results are located.
573 test_name: name of test file (usually a relative path under LayoutTests/)
574 suffix: file suffix of the expected results, including dot; e.g.
575 '.txt' or '.png'. This should not be None, but may be an empty
577 all_baselines: If True, return an ordered list of all baseline paths
578 for the given platform. If False, return only the first one.
580 a list of ( platform_dir, results_filename ), where
581 platform_dir - abs path to the top of the results tree (or test
583 results_filename - relative path from top of tree to the results
585 (port.join() of the two gives you the full path to the file,
586 unless None was returned.)
587 Return values will be in the format appropriate for the current
588 platform (e.g., "\\" for path separators on Windows). If the results
589 file is not found, then None will be returned for the directory,
590 but the expected relative pathname will still be returned.
592 This routine is generic but lives here since it is used in
593 conjunction with the other baseline and filename routines that are
596 baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix
597 baseline_search_path = self.baseline_search_path()
600 for platform_dir in baseline_search_path:
601 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
602 baselines.append((platform_dir, baseline_filename))
604 if not all_baselines and baselines:
607 # If it wasn't found in a platform directory, return the expected
608 # result in the test directory, even if no such file actually exists.
609 platform_dir = self.layout_tests_dir()
610 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
611 baselines.append((platform_dir, baseline_filename))
616 return [(None, baseline_filename)]
618 def expected_filename(self, test_name, suffix, return_default=True):
619 """Given a test name, returns an absolute path to its expected results.
621 If no expected results are found in any of the searched directories,
622 the directory in which the test itself is located will be returned.
623 The return value is in the format appropriate for the platform
624 (e.g., "\\" for path separators on windows).
627 test_name: name of test file (usually a relative path under LayoutTests/)
628 suffix: file suffix of the expected results, including dot; e.g. '.txt'
629 or '.png'. This should not be None, but may be an empty string.
630 platform: the most-specific directory name to use to build the
631 search list of directories, e.g., 'win', or
632 'chromium-cg-mac-leopard' (we follow the WebKit format)
633 return_default: if True, returns the path to the generic expectation if nothing
634 else is found; if False, returns None.
636 This routine is generic but is implemented here to live alongside
637 the other baseline and filename manipulation routines.
639 # FIXME: The [0] here is very mysterious, as is the destructured return.
640 platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0]
642 return self._filesystem.join(platform_dir, baseline_filename)
644 actual_test_name = self.lookup_virtual_test_base(test_name)
646 return self.expected_filename(actual_test_name, suffix)
649 return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
652 def expected_checksum(self, test_name):
653 """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
654 png_path = self.expected_filename(test_name, '.png')
656 if self._filesystem.exists(png_path):
657 with self._filesystem.open_binary_file_for_reading(png_path) as filehandle:
658 return read_checksum_from_png.read_checksum(filehandle)
662 def expected_image(self, test_name):
663 """Returns the image we expect the test to produce."""
664 baseline_path = self.expected_filename(test_name, '.png')
665 if not self._filesystem.exists(baseline_path):
667 return self._filesystem.read_binary_file(baseline_path)
669 def expected_audio(self, test_name):
670 baseline_path = self.expected_filename(test_name, '.wav')
671 if not self._filesystem.exists(baseline_path):
673 return self._filesystem.read_binary_file(baseline_path)
675 def expected_text(self, test_name):
676 """Returns the text output we expect the test to produce, or None
677 if we don't expect there to be any text output.
678 End-of-line characters are normalized to '\n'."""
679 # FIXME: DRT output is actually utf-8, but since we don't decode the
680 # output from DRT (instead treating it as a binary string), we read the
681 # baselines as a binary string, too.
682 baseline_path = self.expected_filename(test_name, '.txt')
683 if not self._filesystem.exists(baseline_path):
685 text = self._filesystem.read_binary_file(baseline_path)
686 return text.replace("\r\n", "\n")
688 def _get_reftest_list(self, test_name):
689 dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
690 if dirname not in self._reftest_list:
691 self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
692 return self._reftest_list[dirname]
695 def _parse_reftest_list(filesystem, test_dirpath):
696 reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
697 if not filesystem.isfile(reftest_list_path):
699 reftest_list_file = filesystem.read_text_file(reftest_list_path)
702 for line in reftest_list_file.split('\n'):
703 line = re.sub('#.+$', '', line)
704 split_line = line.split()
705 if len(split_line) == 4:
706 # FIXME: Probably one of mozilla's extensions in the reftest.list format. Do we need to support this?
707 _log.warning("unsupported reftest.list line '%s' in %s" % (line, reftest_list_path))
709 if len(split_line) < 3:
711 expectation_type, test_file, ref_file = split_line
712 parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
715 def reference_files(self, test_name):
716 """Return a list of expectation (== or !=) and filename pairs"""
718 reftest_list = self._get_reftest_list(test_name)
721 for expectation, prefix in (('==', ''), ('!=', '-mismatch')):
722 for extention in Port._supported_file_extensions:
723 path = self.expected_filename(test_name, prefix + extention)
724 if self._filesystem.exists(path):
725 reftest_list.append((expectation, path))
728 return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), []) # pylint: disable=E1103
730 def tests(self, paths):
731 """Return the list of tests found matching paths."""
732 tests = self._real_tests(paths)
734 suites = self.virtual_test_suites()
736 tests.extend(self._virtual_tests_matching_paths(paths, suites))
738 tests.extend(self._all_virtual_tests(suites))
741 def _real_tests(self, paths):
742 # When collecting test cases, skip these directories
743 skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'support', 'script-tests', 'reference', 'reftest'])
744 files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key)
745 return [self.relative_test_filename(f) for f in files]
747 # When collecting test cases, we include any file with these extensions.
748 _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl',
749 '.htm', '.php', '.svg', '.mht', '.pdf'])
752 # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
753 def is_reference_html_file(filesystem, dirname, filename):
754 if filename.startswith('ref-') or filename.startswith('notref-'):
756 filename_wihout_ext, unused = filesystem.splitext(filename)
757 for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
758 if filename_wihout_ext.endswith(suffix):
763 def _has_supported_extension(filesystem, filename):
764 """Return true if filename is one of the file extensions we want to run a test on."""
765 extension = filesystem.splitext(filename)[1]
766 return extension in Port._supported_file_extensions
769 def is_test_file(filesystem, dirname, filename):
770 return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
772 ALL_TEST_TYPES = ['audio', 'harness', 'pixel', 'ref', 'text', 'unknown']
774 def test_type(self, test_name):
775 fs = self._filesystem
776 if fs.exists(self.expected_filename(test_name, '.png')):
778 if fs.exists(self.expected_filename(test_name, '.wav')):
780 if self.reference_files(test_name):
782 txt = self.expected_text(test_name)
784 if 'layer at (0,0) size 800x600' in txt:
786 for line in txt.splitlines():
787 if line.startswith('FAIL') or line.startswith('TIMEOUT') or line.startswith('PASS'):
792 def test_key(self, test_name):
793 """Turns a test name into a list with two sublists, the natural key of the
794 dirname, and the natural key of the basename.
796 This can be used when sorting paths so that files in a directory.
797 directory are kept together rather than being mixed in with files in
799 dirname, basename = self.split_test(test_name)
800 return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename))
802 def _natural_sort_key(self, string_to_split):
803 """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
805 This can be used to implement "natural sort" order. See:
806 http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
807 http://nedbatchelder.com/blog/200712.html#e20071211T054956
815 return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
818 """Returns the list of top-level test directories."""
819 layout_tests_dir = self.layout_tests_dir()
820 return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
821 self._filesystem.listdir(layout_tests_dir))
824 def test_isfile(self, test_name):
825 """Return True if the test name refers to a directory of tests."""
826 # Used by test_expectations.py to apply rules to whole directories.
827 if self._filesystem.isfile(self.abspath_for_test(test_name)):
829 base = self.lookup_virtual_test_base(test_name)
830 return base and self._filesystem.isfile(self.abspath_for_test(base))
833 def test_isdir(self, test_name):
834 """Return True if the test name refers to a directory of tests."""
835 # Used by test_expectations.py to apply rules to whole directories.
836 if self._filesystem.isdir(self.abspath_for_test(test_name)):
838 base = self.lookup_virtual_test_base(test_name)
839 return base and self._filesystem.isdir(self.abspath_for_test(base))
842 def test_exists(self, test_name):
843 """Return True if the test name refers to an existing test or baseline."""
844 # Used by test_expectations.py to determine if an entry refers to a
845 # valid test and by printing.py to determine if baselines exist.
846 return self.test_isfile(test_name) or self.test_isdir(test_name)
848 def split_test(self, test_name):
849 """Splits a test name into the 'directory' part and the 'basename' part."""
850 index = test_name.rfind(self.TEST_PATH_SEPARATOR)
852 return ('', test_name)
853 return (test_name[0:index], test_name[index:])
855 def normalize_test_name(self, test_name):
856 """Returns a normalized version of the test name or test directory."""
857 if test_name.endswith('/'):
859 if self.test_isdir(test_name):
860 return test_name + '/'
863 def driver_cmd_line(self):
864 """Prints the DRT command line that will be used."""
865 driver = self.create_driver(0)
866 return driver.cmd_line(self.get_option('pixel_tests'), [])
868 def update_baseline(self, baseline_path, data):
869 """Updates the baseline for a test.
872 baseline_path: the actual path to use for baseline, not the path to
873 the test. This function is used to update either generic or
874 platform-specific baselines, but we can't infer which here.
875 data: contents of the baseline.
877 self._filesystem.write_binary_file(baseline_path, data)
879 # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected).
880 def webkit_base(self):
881 return self._webkit_finder.webkit_base()
883 def path_from_webkit_base(self, *comps):
884 return self._webkit_finder.path_from_webkit_base(*comps)
886 def path_from_chromium_base(self, *comps):
887 return self._webkit_finder.path_from_chromium_base(*comps)
889 def path_to_script(self, script_name):
890 return self._webkit_finder.path_to_script(script_name)
892 def layout_tests_dir(self):
893 return self._webkit_finder.layout_tests_dir()
895 def perf_tests_dir(self):
896 return self._webkit_finder.perf_tests_dir()
898 def skipped_layout_tests(self, test_list):
899 """Returns tests skipped outside of the TestExpectations files."""
900 tests = set(self._skipped_tests_for_unsupported_features(test_list))
902 # We explicitly skip any tests in LayoutTests/w3c if need be to avoid running any tests
903 # left over from the old DEPS-pulled repos.
904 # We also will warn at the end of the test run if these directories still exist.
906 # TODO(dpranke): Remove this check after 1/1/2015 and let people deal with the warnings.
907 # Remove the check in controllers/manager.py as well.
908 if self._filesystem.isdir(self._filesystem.join(self.layout_tests_dir(), 'w3c')):
913 def _tests_from_skipped_file_contents(self, skipped_file_contents):
915 for line in skipped_file_contents.split('\n'):
917 line = line.rstrip('/') # Best to normalize directory names to not include the trailing slash.
918 if line.startswith('#') or not len(line):
920 tests_to_skip.append(line)
923 def _expectations_from_skipped_files(self, skipped_file_paths):
925 for search_path in skipped_file_paths:
926 filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped")
927 if not self._filesystem.exists(filename):
928 _log.debug("Skipped does not exist: %s" % filename)
930 _log.debug("Using Skipped file: %s" % filename)
931 skipped_file_contents = self._filesystem.read_text_file(filename)
932 tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
936 def skipped_perf_tests(self):
937 return self._expectations_from_skipped_files([self.perf_tests_dir()])
939 def skips_perf_test(self, test_name):
940 for test_or_category in self.skipped_perf_tests():
941 if test_or_category == test_name:
943 category = self._filesystem.join(self.perf_tests_dir(), test_or_category)
944 if self._filesystem.isdir(category) and test_name.startswith(test_or_category):
948 def is_chromium(self):
952 """Returns a name that uniquely identifies this particular type of port
953 (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed
954 to factory.get() to instantiate the port."""
957 def operating_system(self):
958 # Subclasses should override this default implementation.
962 """Returns a string indicating the version of a given platform, e.g.
965 This is used to help identify the exact port when parsing test
966 expectations, determining search paths, and logging information."""
969 def architecture(self):
970 return self._architecture
972 def get_option(self, name, default_value=None):
973 return getattr(self._options, name, default_value)
975 def set_option_default(self, name, default_value):
976 return self._options.ensure_value(name, default_value)
979 def path_to_generic_test_expectations_file(self):
980 return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations')
982 def relative_test_filename(self, filename):
983 """Returns a test_name a relative unix-style path for a filename under the LayoutTests
984 directory. Ports may legitimately return abspaths here if no relpath makes sense."""
985 # Ports that run on windows need to override this method to deal with
986 # filenames with backslashes in them.
987 if filename.startswith(self.layout_tests_dir()):
988 return self.host.filesystem.relpath(filename, self.layout_tests_dir())
990 return self.host.filesystem.abspath(filename)
993 def abspath_for_test(self, test_name):
994 """Returns the full path to the file for a given test name. This is the
995 inverse of relative_test_filename()."""
996 return self._filesystem.join(self.layout_tests_dir(), test_name)
998 def results_directory(self):
999 """Absolute path to the place to store the test results (uses --results-directory)."""
1000 if not self._results_directory:
1001 option_val = self.get_option('results_directory') or self.default_results_directory()
1002 self._results_directory = self._filesystem.abspath(option_val)
1003 return self._results_directory
1005 def perf_results_directory(self):
1006 return self._build_path()
1008 def default_results_directory(self):
1009 """Absolute path to the default place to store the test results."""
1011 return self.path_from_chromium_base('webkit', self.get_option('configuration'), 'layout-test-results')
1012 except AssertionError:
1013 return self._build_path('layout-test-results')
1015 def setup_test_run(self):
1016 """Perform port-specific work at the beginning of a test run."""
1017 # Delete the disk cache if any to ensure a clean test run.
1018 dump_render_tree_binary_path = self._path_to_driver()
1019 cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
1020 cachedir = self._filesystem.join(cachedir, "cache")
1021 if self._filesystem.exists(cachedir):
1022 self._filesystem.rmtree(cachedir)
1024 if self._dump_reader:
1025 self._filesystem.maybe_make_directory(self._dump_reader.crash_dumps_directory())
1027 def num_workers(self, requested_num_workers):
1028 """Returns the number of available workers (possibly less than the number requested)."""
1029 return requested_num_workers
1031 def clean_up_test_run(self):
1032 """Perform port-specific work at the end of a test run."""
1033 if self._image_differ:
1034 self._image_differ.stop()
1035 self._image_differ = None
1037 # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable.
1038 def _value_or_default_from_environ(self, name, default=None):
1039 if name in os.environ:
1040 return os.environ[name]
1043 def _copy_value_from_environ_if_set(self, clean_env, name):
1044 if name in os.environ:
1045 clean_env[name] = os.environ[name]
1047 def setup_environ_for_server(self, server_name=None):
1048 # We intentionally copy only a subset of os.environ when
1049 # launching subprocesses to ensure consistent test results.
1051 'LOCAL_RESOURCE_ROOT': self.layout_tests_dir(), # FIXME: Is this used?
1053 variables_to_copy = [
1054 'WEBKIT_TESTFONTS', # FIXME: Is this still used?
1055 'WEBKITOUTPUTDIR', # FIXME: Is this still used?
1056 'CHROME_DEVEL_SANDBOX',
1057 'CHROME_IPC_LOGGING',
1060 'VALGRIND_LIB_INNER',
1062 if self.host.platform.is_linux() or self.host.platform.is_freebsd():
1063 variables_to_copy += [
1068 'DBUS_SESSION_BUS_ADDRESS',
1071 clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1')
1072 if self.host.platform.is_mac():
1073 clean_env['DYLD_LIBRARY_PATH'] = self._build_path()
1074 clean_env['DYLD_FRAMEWORK_PATH'] = self._build_path()
1075 variables_to_copy += [
1078 if self.host.platform.is_win():
1079 variables_to_copy += [
1081 'GYP_DEFINES', # Required to locate win sdk.
1083 if self.host.platform.is_cygwin():
1084 variables_to_copy += [
1090 for variable in variables_to_copy:
1091 self._copy_value_from_environ_if_set(clean_env, variable)
1093 for string_variable in self.get_option('additional_env_var', []):
1094 [name, value] = string_variable.split('=', 1)
1095 clean_env[name] = value
1099 def show_results_html_file(self, results_filename):
1100 """This routine should display the HTML file pointed at by
1101 results_filename in a users' browser."""
1102 return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename))
1104 def create_driver(self, worker_number, no_timeout=False):
1105 """Return a newly created Driver subclass for starting/stopping the test driver."""
1106 return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout)
1108 def start_helper(self):
1109 """If a port needs to reconfigure graphics settings or do other
1110 things to ensure a known test configuration, it should override this
1112 helper_path = self._path_to_helper()
1114 _log.debug("Starting layout helper %s" % helper_path)
1115 # Note: Not thread safe: http://bugs.python.org/issue2320
1116 self._helper = self._executive.popen([helper_path],
1117 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
1118 is_ready = self._helper.stdout.readline()
1119 if not is_ready.startswith('ready'):
1120 _log.error("layout_test_helper failed to be ready")
1122 def requires_http_server(self):
1123 """Does the port require an HTTP server for running tests? This could
1124 be the case when the tests aren't run on the host platform."""
1127 def start_http_server(self, additional_dirs, number_of_drivers):
1128 """Start a web server. Raise an error if it can't start or is already running.
1130 Ports can stub this out if they don't need a web server to be running."""
1131 assert not self._http_server, 'Already running an http server.'
1133 server = apache_http.ApacheHTTP(self, self.results_directory(),
1134 additional_dirs=additional_dirs,
1135 number_of_servers=(number_of_drivers * 4))
1137 self._http_server = server
1139 def start_websocket_server(self):
1140 """Start a web server. Raise an error if it can't start or is already running.
1142 Ports can stub this out if they don't need a websocket server to be running."""
1143 assert not self._websocket_server, 'Already running a websocket server.'
1145 server = pywebsocket.PyWebSocket(self, self.results_directory())
1147 self._websocket_server = server
1149 def http_server_supports_ipv6(self):
1150 # Apache < 2.4 on win32 does not support IPv6, nor does cygwin apache.
1151 if self.host.platform.is_cygwin() or self.host.platform.is_win():
1155 def stop_helper(self):
1156 """Shut down the test helper if it is running. Do nothing if
1157 it isn't, or it isn't available. If a port overrides start_helper()
1158 it must override this routine as well."""
1160 _log.debug("Stopping layout test helper")
1162 self._helper.stdin.write("x\n")
1163 self._helper.stdin.close()
1170 def stop_http_server(self):
1171 """Shut down the http server if it is running. Do nothing if it isn't."""
1172 if self._http_server:
1173 self._http_server.stop()
1174 self._http_server = None
1176 def stop_websocket_server(self):
1177 """Shut down the websocket server if it is running. Do nothing if it isn't."""
1178 if self._websocket_server:
1179 self._websocket_server.stop()
1180 self._websocket_server = None
1183 # TEST EXPECTATION-RELATED METHODS
1186 def test_configuration(self):
1187 """Returns the current TestConfiguration for the port."""
1188 if not self._test_configuration:
1189 self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower())
1190 return self._test_configuration
1192 # FIXME: Belongs on a Platform object.
1194 def all_test_configurations(self):
1195 """Returns a list of TestConfiguration instances, representing all available
1196 test configurations for this port."""
1197 return self._generate_all_test_configurations()
1199 # FIXME: Belongs on a Platform object.
1200 def configuration_specifier_macros(self):
1201 """Ports may provide a way to abbreviate configuration specifiers to conveniently
1202 refer to them as one term or alias specific values to more generic ones. For example:
1204 (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake.
1205 (lucid) -> linux # Change specific name of the Linux distro to a more generic term.
1207 Returns a dictionary, each key representing a macro term ('win', for example),
1208 and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7'])."""
1209 return self.CONFIGURATION_SPECIFIER_MACROS
1211 def all_baseline_variants(self):
1212 """Returns a list of platform names sufficient to cover all the baselines.
1214 The list should be sorted so that a later platform will reuse
1215 an earlier platform's baselines if they are the same (e.g.,
1216 'snowleopard' should precede 'leopard')."""
1217 return self.ALL_BASELINE_VARIANTS
1219 def _generate_all_test_configurations(self):
1220 """Returns a sequence of the TestConfigurations the port supports."""
1221 # By default, we assume we want to test every graphics type in
1222 # every configuration on every system.
1223 test_configurations = []
1224 for version, architecture in self.ALL_SYSTEMS:
1225 for build_type in self.ALL_BUILD_TYPES:
1226 test_configurations.append(TestConfiguration(version, architecture, build_type))
1227 return test_configurations
1229 try_builder_names = frozenset([
1238 def warn_if_bug_missing_in_test_expectations(self):
1241 def _port_specific_expectations_files(self):
1243 paths.append(self.path_from_chromium_base('skia', 'skia_test_expectations.txt'))
1244 paths.append(self._filesystem.join(self.layout_tests_dir(), 'NeverFixTests'))
1245 paths.append(self._filesystem.join(self.layout_tests_dir(), 'StaleTestExpectations'))
1246 paths.append(self._filesystem.join(self.layout_tests_dir(), 'SlowTests'))
1247 paths.append(self._filesystem.join(self.layout_tests_dir(), 'FlakyTests'))
1251 def expectations_dict(self):
1252 """Returns an OrderedDict of name -> expectations strings.
1253 The names are expected to be (but not required to be) paths in the filesystem.
1254 If the name is a path, the file can be considered updatable for things like rebaselining,
1255 so don't use names that are paths if they're not paths.
1256 Generally speaking the ordering should be files in the filesystem in cascade order
1257 (TestExpectations followed by Skipped, if the port honors both formats),
1258 then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options."""
1259 # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict.
1260 expectations = OrderedDict()
1262 for path in self.expectations_files():
1263 if self._filesystem.exists(path):
1264 expectations[path] = self._filesystem.read_text_file(path)
1266 for path in self.get_option('additional_expectations', []):
1267 expanded_path = self._filesystem.expanduser(path)
1268 if self._filesystem.exists(expanded_path):
1269 _log.debug("reading additional_expectations from path '%s'" % path)
1270 expectations[path] = self._filesystem.read_text_file(expanded_path)
1272 _log.warning("additional_expectations path '%s' does not exist" % path)
1275 def bot_expectations(self):
1276 if not self.get_option('ignore_flaky_tests'):
1279 full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name)
1280 builder_category = self.get_option('ignore_builder_category', 'layout')
1281 factory = BotTestExpectationsFactory()
1282 # FIXME: This only grabs release builder's flakiness data. If we're running debug,
1283 # when we should grab the debug builder's data.
1284 expectations = factory.expectations_for_port(full_port_name, builder_category)
1286 if not expectations:
1289 ignore_mode = self.get_option('ignore_flaky_tests')
1290 if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky':
1291 return expectations.flakes_by_path(ignore_mode == 'very-flaky')
1292 if ignore_mode == 'unexpected':
1293 return expectations.unexpected_results_by_path()
1294 _log.warning("Unexpected ignore mode: '%s'." % ignore_mode)
1297 def expectations_files(self):
1298 return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files()
1300 def repository_paths(self):
1301 """Returns a list of (repository_name, repository_path) tuples of its depending code base."""
1302 return [('blink', self.layout_tests_dir()),
1303 ('chromium', self.path_from_chromium_base('build'))]
1305 _WDIFF_DEL = '##WDIFF_DEL##'
1306 _WDIFF_ADD = '##WDIFF_ADD##'
1307 _WDIFF_END = '##WDIFF_END##'
1309 def _format_wdiff_output_as_html(self, wdiff):
1310 wdiff = cgi.escape(wdiff)
1311 wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
1312 wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
1313 wdiff = wdiff.replace(self._WDIFF_END, "</span>")
1314 html = "<head><style>.del { background: #faa; } "
1315 html += ".add { background: #afa; }</style></head>"
1316 html += "<pre>%s</pre>" % wdiff
1319 def _wdiff_command(self, actual_filename, expected_filename):
1320 executable = self._path_to_wdiff()
1322 "--start-delete=%s" % self._WDIFF_DEL,
1323 "--end-delete=%s" % self._WDIFF_END,
1324 "--start-insert=%s" % self._WDIFF_ADD,
1325 "--end-insert=%s" % self._WDIFF_END,
1330 def _handle_wdiff_error(script_error):
1331 # Exit 1 means the files differed, any other exit code is an error.
1332 if script_error.exit_code != 1:
1335 def _run_wdiff(self, actual_filename, expected_filename):
1336 """Runs wdiff and may throw exceptions.
1337 This is mostly a hook for unit testing."""
1338 # Diffs are treated as binary as they may include multiple files
1339 # with conflicting encodings. Thus we do not decode the output.
1340 command = self._wdiff_command(actual_filename, expected_filename)
1341 wdiff = self._executive.run_command(command, decode_output=False,
1342 error_handler=self._handle_wdiff_error)
1343 return self._format_wdiff_output_as_html(wdiff)
1345 _wdiff_error_html = "Failed to run wdiff, see error log."
1347 def wdiff_text(self, actual_filename, expected_filename):
1348 """Returns a string of HTML indicating the word-level diff of the
1349 contents of the two filenames. Returns an empty string if word-level
1350 diffing isn't available."""
1351 if not self.wdiff_available():
1354 # It's possible to raise a ScriptError we pass wdiff invalid paths.
1355 return self._run_wdiff(actual_filename, expected_filename)
1356 except OSError as e:
1357 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
1358 # Silently ignore cases where wdiff is missing.
1359 self._wdiff_available = False
1362 except ScriptError as e:
1363 _log.error("Failed to run wdiff: %s" % e)
1364 self._wdiff_available = False
1365 return self._wdiff_error_html
1367 # This is a class variable so we can test error output easily.
1368 _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
1370 def pretty_patch_text(self, diff_path):
1371 if self._pretty_patch_available is None:
1372 self._pretty_patch_available = self.check_pretty_patch(logging=False)
1373 if not self._pretty_patch_available:
1374 return self._pretty_patch_error_html
1375 command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path),
1376 self._pretty_patch_path, diff_path)
1378 # Diffs are treated as binary (we pass decode_output=False) as they
1379 # may contain multiple files of conflicting encodings.
1380 return self._executive.run_command(command, decode_output=False)
1382 # If the system is missing ruby log the error and stop trying.
1383 self._pretty_patch_available = False
1384 _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
1385 return self._pretty_patch_error_html
1386 except ScriptError, e:
1387 # If ruby failed to run for some reason, log the command
1388 # output and stop trying.
1389 self._pretty_patch_available = False
1390 _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output()))
1391 return self._pretty_patch_error_html
1393 def default_configuration(self):
1394 return self._config.default_configuration()
1396 def clobber_old_port_specific_results(self):
1399 # FIXME: This does not belong on the port object.
1401 def path_to_apache(self):
1402 """Returns the full path to the apache binary.
1404 This is needed only by ports that use the apache_http_server module."""
1405 raise NotImplementedError('Port.path_to_apache')
1407 def path_to_apache_config_file(self):
1408 """Returns the full path to the apache configuration file.
1410 If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its
1411 contents will be used instead.
1413 This is needed only by ports that use the apache_http_server module."""
1414 config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH')
1415 if config_file_from_env:
1416 if not self._filesystem.exists(config_file_from_env):
1417 raise IOError('%s was not found on the system' % config_file_from_env)
1418 return config_file_from_env
1420 config_file_name = self._apache_config_file_name_for_platform(sys.platform)
1421 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
1424 # PROTECTED ROUTINES
1426 # The routines below should only be called by routines in this class
1427 # or any of its subclasses.
1430 # FIXME: This belongs on some platform abstraction instead of Port.
1431 def _is_redhat_based(self):
1432 return self._filesystem.exists('/etc/redhat-release')
1434 def _is_debian_based(self):
1435 return self._filesystem.exists('/etc/debian_version')
1437 def _apache_version(self):
1438 config = self._executive.run_command([self.path_to_apache(), '-v'])
1439 return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config)
1441 # We pass sys_platform into this method to make it easy to unit test.
1442 def _apache_config_file_name_for_platform(self, sys_platform):
1443 if sys_platform == 'cygwin':
1444 return 'cygwin-httpd.conf' # CYGWIN is the only platform to still use Apache 1.3.
1445 if sys_platform.startswith('linux'):
1446 if self._is_redhat_based():
1447 return 'fedora-httpd-' + self._apache_version() + '.conf'
1448 if self._is_debian_based():
1449 return 'debian-httpd-' + self._apache_version() + '.conf'
1450 # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
1451 return "apache2-httpd.conf"
1453 def _path_to_driver(self, configuration=None):
1454 """Returns the full path to the test driver."""
1455 return self._build_path(self.driver_name())
1457 def _path_to_webcore_library(self):
1458 """Returns the full path to a built copy of WebCore."""
1461 def _path_to_helper(self):
1462 """Returns the full path to the layout_test_helper binary, which
1463 is used to help configure the system for the test run, or None
1464 if no helper is needed.
1466 This is likely only used by start/stop_helper()."""
1469 def _path_to_image_diff(self):
1470 """Returns the full path to the image_diff binary, or None if it is not available.
1472 This is likely used only by diff_image()"""
1473 return self._build_path('image_diff')
1476 def _path_to_wdiff(self):
1477 """Returns the full path to the wdiff binary, or None if it is not available.
1479 This is likely used only by wdiff_text()"""
1480 for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"):
1481 if self._filesystem.exists(path):
1485 def _webkit_baseline_path(self, platform):
1486 """Return the full path to the top of the baseline tree for a
1488 return self._filesystem.join(self.layout_tests_dir(), 'platform', platform)
1490 def _driver_class(self):
1491 """Returns the port's driver implementation."""
1492 return driver.Driver
1494 def _output_contains_sanitizer_messages(self, output):
1497 if 'AddressSanitizer' in output:
1498 return 'AddressSanitizer'
1499 if 'MemorySanitizer' in output:
1500 return 'MemorySanitizer'
1503 def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
1504 if self._output_contains_sanitizer_messages(stderr):
1505 # Running the symbolizer script can take a lot of memory, so we need to
1506 # serialize access to it across all the concurrently running drivers.
1508 llvm_symbolizer_path = self.path_from_chromium_base('third_party', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer')
1509 if self._filesystem.exists(llvm_symbolizer_path):
1510 env = os.environ.copy()
1511 env['LLVM_SYMBOLIZER_PATH'] = llvm_symbolizer_path
1514 sanitizer_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
1515 sanitizer_strip_path_prefix = 'Release/../../'
1516 if self._filesystem.exists(sanitizer_filter_path):
1517 stderr = self._executive.run_command(['flock', sys.executable, sanitizer_filter_path, sanitizer_strip_path_prefix], input=stderr, decode_output=False, env=env)
1519 name_str = name or '<unknown process name>'
1520 pid_str = str(pid or '<unknown>')
1521 stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines()
1522 stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines()
1523 return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str,
1524 '\n'.join(('STDOUT: ' + l) for l in stdout_lines),
1525 '\n'.join(('STDERR: ' + l) for l in stderr_lines)))
1527 def look_for_new_crash_logs(self, crashed_processes, start_time):
1530 def look_for_new_samples(self, unresponsive_processes, start_time):
1533 def sample_process(self, name, pid):
1536 def physical_test_suites(self):
1538 # For example, to turn on force-compositing-mode in the svg/ directory:
1539 # PhysicalTestSuite('svg',
1540 # ['--force-compositing-mode']),
1543 def virtual_test_suites(self):
1544 if self._virtual_test_suites is None:
1545 path_to_virtual_test_suites = self._filesystem.join(self.layout_tests_dir(), 'VirtualTestSuites')
1546 assert self._filesystem.exists(path_to_virtual_test_suites), 'LayoutTests/VirtualTestSuites not found'
1548 test_suite_json = json.loads(self._filesystem.read_text_file(path_to_virtual_test_suites))
1549 self._virtual_test_suites = [VirtualTestSuite(**d) for d in test_suite_json]
1550 except ValueError as e:
1551 raise ValueError("LayoutTests/VirtualTestSuites is not a valid JSON file: %s" % str(e))
1552 return self._virtual_test_suites
1554 def _all_virtual_tests(self, suites):
1556 for suite in suites:
1557 self._populate_virtual_suite(suite)
1558 tests.extend(suite.tests.keys())
1561 def _virtual_tests_matching_paths(self, paths, suites):
1563 for suite in suites:
1564 if any(p.startswith(suite.name) for p in paths):
1565 self._populate_virtual_suite(suite)
1566 for test in suite.tests:
1567 if any(test.startswith(p) for p in paths):
1571 def _populate_virtual_suite(self, suite):
1573 base_tests = self._real_tests([suite.base])
1575 for test in base_tests:
1576 suite.tests[test.replace(suite.base, suite.name, 1)] = test
1578 def is_virtual_test(self, test_name):
1579 return bool(self.lookup_virtual_suite(test_name))
1581 def lookup_virtual_suite(self, test_name):
1582 for suite in self.virtual_test_suites():
1583 if test_name.startswith(suite.name):
1587 def lookup_virtual_test_base(self, test_name):
1588 suite = self.lookup_virtual_suite(test_name)
1591 return test_name.replace(suite.name, suite.base, 1)
1593 def lookup_virtual_test_args(self, test_name):
1594 for suite in self.virtual_test_suites():
1595 if test_name.startswith(suite.name):
1599 def lookup_physical_test_args(self, test_name):
1600 for suite in self.physical_test_suites():
1601 if test_name.startswith(suite.name):
1605 def should_run_as_pixel_test(self, test_input):
1606 if not self._options.pixel_tests:
1608 if self._options.pixel_test_directories:
1609 return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories)
1612 def _modules_to_search_for_symbols(self):
1613 path = self._path_to_webcore_library()
1618 def _symbols_string(self):
1620 for path_to_module in self._modules_to_search_for_symbols():
1622 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error)
1624 _log.warn("Failed to run nm: %s. Can't determine supported features correctly." % e)
1627 # Ports which use compile-time feature detection should define this method and return
1628 # a dictionary mapping from symbol substrings to possibly disabled test directories.
1629 # When the symbol substrings are not matched, the directories will be skipped.
1630 # If ports don't ever enable certain features, then those directories can just be
1631 # in the Skipped list instead of compile-time-checked here.
1632 def _missing_symbol_to_skipped_tests(self):
1633 if self.PORT_HAS_AUDIO_CODECS_BUILT_IN:
1637 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"],
1638 "ff_aac_decoder": ["webaudio/codec-tests/aac"],
1641 def _has_test_in_directories(self, directory_lists, test_list):
1645 directories = itertools.chain.from_iterable(directory_lists)
1646 for directory, test in itertools.product(directories, test_list):
1647 if test.startswith(directory):
1651 def _skipped_tests_for_unsupported_features(self, test_list):
1652 # Only check the symbols of there are tests in the test_list that might get skipped.
1653 # This is a performance optimization to avoid the calling nm.
1654 # Runtime feature detection not supported, fallback to static detection:
1655 # Disable any tests for symbols missing from the executable or libraries.
1656 if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
1657 symbols_string = self._symbols_string()
1658 if symbols_string is not None:
1659 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], [])
1662 def _convert_path(self, path):
1663 """Handles filename conversion for subprocess command line args."""
1664 # See note above in diff_image() for why we need this.
1665 if sys.platform == 'cygwin':
1666 return cygpath(path)
1669 def _build_path(self, *comps):
1670 return self._build_path_with_configuration(None, *comps)
1672 def _build_path_with_configuration(self, configuration, *comps):
1673 # Note that we don't do the option caching that the
1674 # base class does, because finding the right directory is relatively
1676 configuration = configuration or self.get_option('configuration')
1677 return self._static_build_path(self._filesystem, self.get_option('build_directory'),
1678 self.path_from_chromium_base(), configuration, comps)
1680 def _check_driver_build_up_to_date(self, configuration):
1681 if configuration in ('Debug', 'Release'):
1683 debug_path = self._path_to_driver('Debug')
1684 release_path = self._path_to_driver('Release')
1686 debug_mtime = self._filesystem.mtime(debug_path)
1687 release_mtime = self._filesystem.mtime(release_path)
1689 if (debug_mtime > release_mtime and configuration == 'Release' or
1690 release_mtime > debug_mtime and configuration == 'Debug'):
1691 most_recent_binary = 'Release' if configuration == 'Debug' else 'Debug'
1692 _log.warning('You are running the %s binary. However the %s binary appears to be more recent. '
1693 'Please pass --%s.', configuration, most_recent_binary, most_recent_binary.lower())
1695 # This will fail if we don't have both a debug and release binary.
1696 # That's fine because, in this case, we must already be running the
1697 # most up-to-date one.
1702 def _chromium_baseline_path(self, platform):
1703 if platform is None:
1704 platform = self.name()
1705 return self.path_from_webkit_base('LayoutTests', 'platform', platform)
1707 class VirtualTestSuite(object):
1708 def __init__(self, prefix=None, base=None, args=None):
1711 assert prefix.find('/') == -1, "Virtual test suites prefixes cannot contain /'s: %s" % prefix
1712 self.name = 'virtual/' + prefix + '/' + base
1718 return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)
1721 class PhysicalTestSuite(object):
1722 def __init__(self, base, args):
1729 return "PhysicalTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)